From 6e1034996e0a65f22407d55bd94e1458a98abae0 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Fri, 16 Jun 2017 16:43:39 +0200 Subject: [PATCH 1/3] UTF8Encoding: introduce new lib. --- code/BaseImporter.cpp | 44 +- code/SIBImporter.cpp | 31 +- contrib/utf8cpp/doc/ReleaseNotes | 12 + contrib/utf8cpp/doc/utf8cpp.html | 1789 +++++++++++++++++++++++ contrib/utf8cpp/source/utf8.h | 34 + contrib/utf8cpp/source/utf8/checked.h | 327 +++++ contrib/utf8cpp/source/utf8/core.h | 329 +++++ contrib/utf8cpp/source/utf8/unchecked.h | 228 +++ 8 files changed, 2768 insertions(+), 26 deletions(-) create mode 100644 contrib/utf8cpp/doc/ReleaseNotes create mode 100644 contrib/utf8cpp/doc/utf8cpp.html create mode 100644 contrib/utf8cpp/source/utf8.h create mode 100644 contrib/utf8cpp/source/utf8/checked.h create mode 100644 contrib/utf8cpp/source/utf8/core.h create mode 100644 contrib/utf8cpp/source/utf8/unchecked.h diff --git a/code/BaseImporter.cpp b/code/BaseImporter.cpp index e8d547783..ba5faed80 100644 --- a/code/BaseImporter.cpp +++ b/code/BaseImporter.cpp @@ -303,10 +303,11 @@ void BaseImporter::GetExtensionList(std::set& extensions) return false; } -#include "../contrib/ConvertUTF/ConvertUTF.h" +//#include "../contrib/ConvertUTF/ConvertUTF.h" +#include "../contrib/utf8cpp/source/utf8.h" // ------------------------------------------------------------------------------------------------ -void ReportResult(ConversionResult res) +/*void ReportResult(ConversionResult res) { if(res == sourceExhausted) { DefaultLogger::get()->error("Source ends with incomplete character sequence, transformation to UTF-8 fails"); @@ -314,13 +315,13 @@ void ReportResult(ConversionResult res) else if(res == sourceIllegal) { DefaultLogger::get()->error("Source contains illegal character sequence, transformation to UTF-8 fails"); } -} +}*/ // ------------------------------------------------------------------------------------------------ // Convert to UTF8 data void BaseImporter::ConvertToUTF8(std::vector& data) { - ConversionResult result; + //ConversionResult result; if(data.size() < 8) { throw DeadlyImportError("File is too small"); } @@ -333,7 +334,8 @@ void BaseImporter::ConvertToUTF8(std::vector& data) data.resize(data.size()-3); return; } - + + // UTF 32 BE with BOM if(*((uint32_t*)&data.front()) == 0xFFFE0000) { @@ -348,20 +350,23 @@ void BaseImporter::ConvertToUTF8(std::vector& data) DefaultLogger::get()->debug("Found UTF-32 BOM ..."); const uint32_t* sstart = (uint32_t*)&data.front()+1, *send = (uint32_t*)&data.back()+1; - char* dstart,*dend; +// char* dstart,*dend; std::vector output; - do { + int *ptr = (int*)&data[ 0 ]; + int *end = ptr + ( data.size() / sizeof(int) ) +1; + utf8::utf32to8( ptr, end, back_inserter(output)); +/* do { output.resize(output.size()?output.size()*3/2:data.size()/2); dstart = &output.front(),dend = &output.back()+1; - result = ConvertUTF32toUTF8((const UTF32**)&sstart,(const UTF32*)send,(UTF8**)&dstart,(UTF8*)dend,lenientConversion); - } while(result == targetExhausted); + ///result = ConvertUTF32toUTF8((const UTF32**)&sstart,(const UTF32*)send,(UTF8**)&dstart,(UTF8*)dend,lenientConversion); + } while(result == targetExhausted);*/ - ReportResult(result); + // ReportResult(result); // copy to output buffer. - const size_t outlen = (size_t)(dstart-&output.front()); - data.assign(output.begin(),output.begin()+outlen); +// const size_t outlen = (size_t)(dstart-&output.front()); +// data.assign(output.begin(),output.begin()+outlen); return; } @@ -379,13 +384,18 @@ void BaseImporter::ConvertToUTF8(std::vector& data) DefaultLogger::get()->debug("Found UTF-16 BOM ..."); const uint16_t* sstart = (uint16_t*)&data.front()+1, *send = (uint16_t*)(&data.back()+1); - char* dstart,*dend; - std::vector output; - do { +// char* dstart,*dend; + std::vector output; + int16_t *ptr = (int16_t*) &data[ 0 ]; + int16_t *end = ptr + (data.size() / sizeof(int)) + 1; + + utf8::utf16to8(data.begin(), data.end(), back_inserter(output)); +/* do { output.resize(output.size()?output.size()*3/2:data.size()*3/4); dstart = &output.front(),dend = &output.back()+1; - result = ConvertUTF16toUTF8((const UTF16**)&sstart,(const UTF16*)send,(UTF8**)&dstart,(UTF8*)dend,lenientConversion); + utf8::utf16to8(data.begin(), data.end(), output); + } while(result == targetExhausted); ReportResult(result); @@ -393,7 +403,7 @@ void BaseImporter::ConvertToUTF8(std::vector& data) // copy to output buffer. const size_t outlen = (size_t)(dstart-&output.front()); data.assign(output.begin(),output.begin()+outlen); - return; + return;*/ } } diff --git a/code/SIBImporter.cpp b/code/SIBImporter.cpp index 87e0d1585..6cd398f48 100644 --- a/code/SIBImporter.cpp +++ b/code/SIBImporter.cpp @@ -58,7 +58,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "ByteSwapper.h" #include "StreamReader.h" #include "TinyFormatter.h" -#include "../contrib/ConvertUTF/ConvertUTF.h" +//#include "../contrib/ConvertUTF/ConvertUTF.h" +#include "../contrib/utf8cpp/source/utf8.h" #include #include #include @@ -177,21 +178,33 @@ static void UnknownChunk(StreamReaderLE* stream, const SIBChunk& chunk) // Reads a UTF-16LE string and returns it at UTF-8. static aiString ReadString(StreamReaderLE* stream, uint32_t numWChars) { + if ( 0 == numWChars ) { + static const aiString empty; + return empty; + } // Allocate buffers (max expansion is 1 byte -> 4 bytes for UTF-8) - UTF16* temp = new UTF16[numWChars]; - UTF8* str = new UTF8[numWChars * 4 + 1]; + //UTF16* temp = new UTF16[numWChars]; + std::vector str; + str.reserve(numWChars * 4 + 1); + //unsigned char* str = new unsigned char[numWChars * 4 + 1]; + uint16_t *temp = new uint16_t[numWChars]; for (uint32_t n=0;nGetU2(); // Convert it and NUL-terminate. - const UTF16 *start = temp, *end = temp + numWChars; - UTF8 *dest = str, *limit = str + numWChars*4; - ConvertUTF16toUTF8(&start, end, &dest, limit, lenientConversion); - *dest = '\0'; + //const UTF16 *start = temp, *end = temp + numWChars; + const uint16_t *start = temp, *end = temp + numWChars; + utf8::utf16to8(start, end, back_inserter(str)); + + //UTF8 *dest = str, *limit = str + numWChars*4; + //ConvertUTF16toUTF8(&start, end, &dest, limit, lenientConversion); + //*dest = '\0'; + + str[str.size()-1] = '\0'; // Return the final string. - aiString result = aiString((const char *)str); - delete[] str; + aiString result = aiString((const char *)&str[0]); + //delete[] str; delete[] temp; return result; } diff --git a/contrib/utf8cpp/doc/ReleaseNotes b/contrib/utf8cpp/doc/ReleaseNotes new file mode 100644 index 000000000..364411a23 --- /dev/null +++ b/contrib/utf8cpp/doc/ReleaseNotes @@ -0,0 +1,12 @@ +utf8 cpp library +Release 2.3.4 + +A minor bug fix release. Thanks to all who reported bugs. + +Note: Version 2.3.3 contained a regression, and therefore was removed. + +Changes from version 2.3.2 +- Bug fix [39]: checked.h Line 273 and unchecked.h Line 182 have an extra ';' +- Bug fix [36]: replace_invalid() only works with back_inserter + +Files included in the release: utf8.h, core.h, checked.h, unchecked.h, utf8cpp.html, ReleaseNotes diff --git a/contrib/utf8cpp/doc/utf8cpp.html b/contrib/utf8cpp/doc/utf8cpp.html new file mode 100644 index 000000000..6f2aacbe7 --- /dev/null +++ b/contrib/utf8cpp/doc/utf8cpp.html @@ -0,0 +1,1789 @@ + + + + + + + + + UTF8-CPP: UTF-8 with C++ in a Portable Way + + + + +

+ UTF8-CPP: UTF-8 with C++ in a Portable Way +

+

+ The Sourceforge project page +

+ +

+ Introduction +

+

+ Many C++ developers miss an easy and portable way of handling Unicode encoded + strings. The original C++ Standard (known as C++98 or C++03) is Unicode agnostic. + C++11 provides some support for Unicode on core language and library level: + u8, u, and U character and string literals, char16_t and char32_t character types, + u16string and u32string library classes, and codecvt support for conversions + between Unicode encoding forms. + In the meantime, developers use third party libraries like ICU, OS specific capabilities, or simply + roll out their own solutions. +

+

+ In order to easily handle UTF-8 encoded Unicode strings, I came up with a small + generic library. For anybody used to work with STL algorithms and iterators, it should be + easy and natural to use. The code is freely available for any purpose - check out + the license at the beginning of the utf8.h file. If you run into + bugs or performance issues, please let me know and I'll do my best to address them. +

+

+ The purpose of this article is not to offer an introduction to Unicode in general, + and UTF-8 in particular. If you are not familiar with Unicode, be sure to check out + Unicode Home Page or some other source of + information for Unicode. Also, it is not my aim to advocate the use of UTF-8 + encoded strings in C++ programs; if you want to handle UTF-8 encoded strings from + C++, I am sure you have good reasons for it. +

+

+ Examples of use +

+

+ Introductionary Sample +

+

+ To illustrate the use of the library, let's start with a small but complete program + that opens a file containing UTF-8 encoded text, reads it line by line, checks each line + for invalid UTF-8 byte sequences, and converts it to UTF-16 encoding and back to UTF-8: +

+
+#include <fstream>
+#include <iostream>
+#include <string>
+#include <vector>
+#include "utf8.h"
+using namespace std;
+int main(int argc, char** argv)
+{
+    if (argc != 2) {
+        cout << "\nUsage: docsample filename\n";
+        return 0;
+    }
+
+    const char* test_file_path = argv[1];
+    // Open the test file (contains UTF-8 encoded text)
+    ifstream fs8(test_file_path);
+    if (!fs8.is_open()) {
+    cout << "Could not open " << test_file_path << endl;
+    return 0;
+    }
+
+    unsigned line_count = 1;
+    string line;
+    // Play with all the lines in the file
+    while (getline(fs8, line)) {
+       // check for invalid utf-8 (for a simple yes/no check, there is also utf8::is_valid function)
+        string::iterator end_it = utf8::find_invalid(line.begin(), line.end());
+        if (end_it != line.end()) {
+            cout << "Invalid UTF-8 encoding detected at line " << line_count << "\n";
+            cout << "This part is fine: " << string(line.begin(), end_it) << "\n";
+        }
+
+        // Get the line length (at least for the valid part)
+        int length = utf8::distance(line.begin(), end_it);
+        cout << "Length of line " << line_count << " is " << length <<  "\n";
+
+        // Convert it to utf-16
+        vector<unsigned short> utf16line;
+        utf8::utf8to16(line.begin(), end_it, back_inserter(utf16line));
+
+        // And back to utf-8
+        string utf8line; 
+        utf8::utf16to8(utf16line.begin(), utf16line.end(), back_inserter(utf8line));
+
+        // Confirm that the conversion went OK:
+        if (utf8line != string(line.begin(), end_it))
+            cout << "Error in UTF-16 conversion at line: " << line_count << "\n";        
+
+        line_count++;
+    }
+    return 0;
+}
+
+

+ In the previous code sample, for each line we performed + a detection of invalid UTF-8 sequences with find_invalid; the number + of characters (more precisely - the number of Unicode code points, including the end + of line and even BOM if there is one) in each line was + determined with a use of utf8::distance; finally, we have converted + each line to UTF-16 encoding with utf8to16 and back to UTF-8 with + utf16to8. +

+

Checking if a file contains valid UTF-8 text

+

+Here is a function that checks whether the content of a file is valid UTF-8 encoded text without +reading the content into the memory: +

+
    
+bool valid_utf8_file(iconst char* file_name)
+{
+    ifstream ifs(file_name);
+    if (!ifs)
+        return false; // even better, throw here
+
+    istreambuf_iterator<char> it(ifs.rdbuf());
+    istreambuf_iterator<char> eos;
+
+    return utf8::is_valid(it, eos);
+}
+
+

+Because the function utf8::is_valid() works with input iterators, we were able +to pass an istreambuf_iterator to it and read the content of the file directly +without loading it to the memory first.

+

+Note that other functions that take input iterator arguments can be used in a similar way. For +instance, to read the content of a UTF-8 encoded text file and convert the text to UTF-16, just +do something like: +

+
+    utf8::utf8to16(it, eos, back_inserter(u16string));
+
+

Ensure that a string contains valid UTF-8 text

+

+If we have some text that "probably" contains UTF-8 encoded text and we want to +replace any invalid UTF-8 sequence with a replacement character, something like +the following function may be used: +

+
+void fix_utf8_string(std::string& str)
+{
+    std::string temp;
+    utf8::replace_invalid(str.begin(), str.end(), back_inserter(temp));
+    str = temp;
+}
+
+

The function will replace any invalid UTF-8 sequence with a Unicode replacement character. +There is an overloaded function that enables the caller to supply their own replacement character. +

+

+ Reference +

+

+ Functions From utf8 Namespace +

+

+ utf8::append +

+

+ Available in version 1.0 and later. +

+

+ Encodes a 32 bit code point as a UTF-8 sequence of octets and appends the sequence + to a UTF-8 string. +

+
+template <typename octet_iterator>
+octet_iterator append(uint32_t cp, octet_iterator result);
+   
+
+

+ octet_iterator: an output iterator.
+ cp: a 32 bit integer representing a code point to append to the + sequence.
+ result: an output iterator to the place in the sequence where to + append the code point.
+ Return value: an iterator pointing to the place + after the newly appended sequence. +

+

+ Example of use: +

+
+unsigned char u[5] = {0,0,0,0,0};
+unsigned char* end = append(0x0448, u);
+assert (u[0] == 0xd1 && u[1] == 0x88 && u[2] == 0 && u[3] == 0 && u[4] == 0);
+
+

+ Note that append does not allocate any memory - it is the burden of + the caller to make sure there is enough memory allocated for the operation. To make + things more interesting, append can add anywhere between 1 and 4 + octets to the sequence. In practice, you would most often want to use + std::back_inserter to ensure that the necessary memory is allocated. +

+

+ In case of an invalid code point, a utf8::invalid_code_point exception + is thrown. +

+

+ utf8::next +

+

+ Available in version 1.0 and later. +

+

+ Given the iterator to the beginning of the UTF-8 sequence, it returns the code + point and moves the iterator to the next position. +

+
+template <typename octet_iterator> 
+uint32_t next(octet_iterator& it, octet_iterator end);
+   
+
+

+ octet_iterator: an input iterator.
+ it: a reference to an iterator pointing to the beginning of an UTF-8 + encoded code point. After the function returns, it is incremented to point to the + beginning of the next code point.
+ end: end of the UTF-8 sequence to be processed. If it + gets equal to end during the extraction of a code point, an + utf8::not_enough_room exception is thrown.
+ Return value: the 32 bit representation of the + processed UTF-8 code point. +

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+char* w = twochars;
+int cp = next(w, twochars + 6);
+assert (cp == 0x65e5);
+assert (w == twochars + 3);
+
+

+ This function is typically used to iterate through a UTF-8 encoded string. +

+

+ In case of an invalid UTF-8 seqence, a utf8::invalid_utf8 exception is + thrown. +

+

+ utf8::peek_next +

+

+ Available in version 2.1 and later. +

+

+ Given the iterator to the beginning of the UTF-8 sequence, it returns the code + point for the following sequence without changing the value of the iterator. +

+
+template <typename octet_iterator> 
+uint32_t peek_next(octet_iterator it, octet_iterator end);
+   
+
+

+ octet_iterator: an input iterator.
+ it: an iterator pointing to the beginning of an UTF-8 + encoded code point.
+ end: end of the UTF-8 sequence to be processed. If it + gets equal to end during the extraction of a code point, an + utf8::not_enough_room exception is thrown.
+ Return value: the 32 bit representation of the + processed UTF-8 code point. +

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+char* w = twochars;
+int cp = peek_next(w, twochars + 6);
+assert (cp == 0x65e5);
+assert (w == twochars);
+
+

+ In case of an invalid UTF-8 seqence, a utf8::invalid_utf8 exception is + thrown. +

+

+ utf8::prior +

+

+ Available in version 1.02 and later. +

+

+ Given a reference to an iterator pointing to an octet in a UTF-8 sequence, it + decreases the iterator until it hits the beginning of the previous UTF-8 encoded + code point and returns the 32 bits representation of the code point. +

+
+template <typename octet_iterator> 
+uint32_t prior(octet_iterator& it, octet_iterator start);
+   
+
+

+ octet_iterator: a bidirectional iterator.
+ it: a reference pointing to an octet within a UTF-8 encoded string. + After the function returns, it is decremented to point to the beginning of the + previous code point.
+ start: an iterator to the beginning of the sequence where the search + for the beginning of a code point is performed. It is a + safety measure to prevent passing the beginning of the string in the search for a + UTF-8 lead octet.
+ Return value: the 32 bit representation of the + previous code point. +

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+unsigned char* w = twochars + 3;
+int cp = prior (w, twochars);
+assert (cp == 0x65e5);
+assert (w == twochars);
+
+

+ This function has two purposes: one is two iterate backwards through a UTF-8 + encoded string. Note that it is usually a better idea to iterate forward instead, + since utf8::next is faster. The second purpose is to find a beginning + of a UTF-8 sequence if we have a random position within a string. Note that in that + case utf8::prior may not detect an invalid UTF-8 sequence in some scenarios: + for instance if there are superfluous trail octets, it will just skip them. +

+

+ it will typically point to the beginning of + a code point, and start will point to the + beginning of the string to ensure we don't go backwards too far. it is + decreased until it points to a lead UTF-8 octet, and then the UTF-8 sequence + beginning with that octet is decoded to a 32 bit representation and returned. +

+

+ In case start is reached before a UTF-8 lead octet is hit, or if an + invalid UTF-8 sequence is started by the lead octet, an invalid_utf8 + exception is thrown. +

+

In case start equals it, a not_enough_room + exception is thrown. +

+ utf8::previous +

+

+ Deprecated in version 1.02 and later. +

+

+ Given a reference to an iterator pointing to an octet in a UTF-8 seqence, it + decreases the iterator until it hits the beginning of the previous UTF-8 encoded + code point and returns the 32 bits representation of the code point. +

+
+template <typename octet_iterator> 
+uint32_t previous(octet_iterator& it, octet_iterator pass_start);
+   
+
+

+ octet_iterator: a random access iterator.
+ it: a reference pointing to an octet within a UTF-8 encoded string. + After the function returns, it is decremented to point to the beginning of the + previous code point.
+ pass_start: an iterator to the point in the sequence where the search + for the beginning of a code point is aborted if no result was reached. It is a + safety measure to prevent passing the beginning of the string in the search for a + UTF-8 lead octet.
+ Return value: the 32 bit representation of the + previous code point. +

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+unsigned char* w = twochars + 3;
+int cp = previous (w, twochars - 1);
+assert (cp == 0x65e5);
+assert (w == twochars);
+
+

+ utf8::previous is deprecated, and utf8::prior should + be used instead, although the existing code can continue using this function. + The problem is the parameter pass_start that points to the position + just before the beginning of the sequence. Standard containers don't have the + concept of "pass start" and the function can not be used with their iterators. +

+

+ it will typically point to the beginning of + a code point, and pass_start will point to the octet just before the + beginning of the string to ensure we don't go backwards too far. it is + decreased until it points to a lead UTF-8 octet, and then the UTF-8 sequence + beginning with that octet is decoded to a 32 bit representation and returned. +

+

+ In case pass_start is reached before a UTF-8 lead octet is hit, or if an + invalid UTF-8 sequence is started by the lead octet, an invalid_utf8 + exception is thrown +

+

+ utf8::advance +

+

+ Available in version 1.0 and later. +

+

+ Advances an iterator by the specified number of code points within an UTF-8 + sequence. +

+
+template <typename octet_iterator, typename distance_type> 
+void advance (octet_iterator& it, distance_type n, octet_iterator end);
+   
+
+

+ octet_iterator: an input iterator.
+ distance_type: an integral type convertible to octet_iterator's difference type.
+ it: a reference to an iterator pointing to the beginning of an UTF-8 + encoded code point. After the function returns, it is incremented to point to the + nth following code point.
+ n: a positive integer that shows how many code points we want to + advance.
+ end: end of the UTF-8 sequence to be processed. If it + gets equal to end during the extraction of a code point, an + utf8::not_enough_room exception is thrown.
+

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+unsigned char* w = twochars;
+advance (w, 2, twochars + 6);
+assert (w == twochars + 5);
+
+

+ This function works only "forward". In case of a negative n, there is + no effect. +

+

+ In case of an invalid code point, a utf8::invalid_code_point exception + is thrown. +

+

+ utf8::distance +

+

+ Available in version 1.0 and later. +

+

+ Given the iterators to two UTF-8 encoded code points in a seqence, returns the + number of code points between them. +

+
+template <typename octet_iterator> 
+typename std::iterator_traits<octet_iterator>::difference_type distance (octet_iterator first, octet_iterator last);
+   
+
+

+ octet_iterator: an input iterator.
+ first: an iterator to a beginning of a UTF-8 encoded code point.
+ last: an iterator to a "post-end" of the last UTF-8 encoded code + point in the sequence we are trying to determine the length. It can be the + beginning of a new code point, or not.
+ Return value the distance between the iterators, + in code points. +

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+size_t dist = utf8::distance(twochars, twochars + 5);
+assert (dist == 2);
+
+

+ This function is used to find the length (in code points) of a UTF-8 encoded + string. The reason it is called distance, rather than, say, + length is mainly because developers are used that length is an + O(1) function. Computing the length of an UTF-8 string is a linear operation, and + it looked better to model it after std::distance algorithm. +

+

+ In case of an invalid UTF-8 seqence, a utf8::invalid_utf8 exception is + thrown. If last does not point to the past-of-end of a UTF-8 seqence, + a utf8::not_enough_room exception is thrown. +

+

+ utf8::utf16to8 +

+

+ Available in version 1.0 and later. +

+

+ Converts a UTF-16 encoded string to UTF-8. +

+
+template <typename u16bit_iterator, typename octet_iterator>
+octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result);
+   
+
+

+ u16bit_iterator: an input iterator.
+ octet_iterator: an output iterator.
+ start: an iterator pointing to the beginning of the UTF-16 encoded + string to convert.
+ end: an iterator pointing to pass-the-end of the UTF-16 encoded + string to convert.
+ result: an output iterator to the place in the UTF-8 string where to + append the result of conversion.
+ Return value: An iterator pointing to the place + after the appended UTF-8 string. +

+

+ Example of use: +

+
+unsigned short utf16string[] = {0x41, 0x0448, 0x65e5, 0xd834, 0xdd1e};
+vector<unsigned char> utf8result;
+utf16to8(utf16string, utf16string + 5, back_inserter(utf8result));
+assert (utf8result.size() == 10);    
+
+

+ In case of invalid UTF-16 sequence, a utf8::invalid_utf16 exception is + thrown. +

+

+ utf8::utf8to16 +

+

+ Available in version 1.0 and later. +

+

+ Converts an UTF-8 encoded string to UTF-16 +

+
+template <typename u16bit_iterator, typename octet_iterator>
+u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result);
+   
+
+

+ octet_iterator: an input iterator.
+ u16bit_iterator: an output iterator.
+ start: an iterator pointing to the beginning of the UTF-8 encoded + string to convert. < br /> end: an iterator pointing to + pass-the-end of the UTF-8 encoded string to convert.
+ result: an output iterator to the place in the UTF-16 string where to + append the result of conversion.
+ Return value: An iterator pointing to the place + after the appended UTF-16 string. +

+

+ Example of use: +

+
+char utf8_with_surrogates[] = "\xe6\x97\xa5\xd1\x88\xf0\x9d\x84\x9e";
+vector <unsigned short> utf16result;
+utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result));
+assert (utf16result.size() == 4);
+assert (utf16result[2] == 0xd834);
+assert (utf16result[3] == 0xdd1e);
+
+

+ In case of an invalid UTF-8 seqence, a utf8::invalid_utf8 exception is + thrown. If end does not point to the past-of-end of a UTF-8 seqence, a + utf8::not_enough_room exception is thrown. +

+

+ utf8::utf32to8 +

+

+ Available in version 1.0 and later. +

+

+ Converts a UTF-32 encoded string to UTF-8. +

+
+template <typename octet_iterator, typename u32bit_iterator>
+octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result);
+   
+
+

+ octet_iterator: an output iterator.
+ u32bit_iterator: an input iterator.
+ start: an iterator pointing to the beginning of the UTF-32 encoded + string to convert.
+ end: an iterator pointing to pass-the-end of the UTF-32 encoded + string to convert.
+ result: an output iterator to the place in the UTF-8 string where to + append the result of conversion.
+ Return value: An iterator pointing to the place + after the appended UTF-8 string. +

+

+ Example of use: +

+
+int utf32string[] = {0x448, 0x65E5, 0x10346, 0};
+vector<unsigned char> utf8result;
+utf32to8(utf32string, utf32string + 3, back_inserter(utf8result));
+assert (utf8result.size() == 9);
+
+

+ In case of invalid UTF-32 string, a utf8::invalid_code_point exception + is thrown. +

+

+ utf8::utf8to32 +

+

+ Available in version 1.0 and later. +

+

+ Converts a UTF-8 encoded string to UTF-32. +

+
+template <typename octet_iterator, typename u32bit_iterator>
+u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result);
+   
+
+

+ octet_iterator: an input iterator.
+ u32bit_iterator: an output iterator.
+ start: an iterator pointing to the beginning of the UTF-8 encoded + string to convert.
+ end: an iterator pointing to pass-the-end of the UTF-8 encoded string + to convert.
+ result: an output iterator to the place in the UTF-32 string where to + append the result of conversion.
+ Return value: An iterator pointing to the place + after the appended UTF-32 string. +

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+vector<int> utf32result;
+utf8to32(twochars, twochars + 5, back_inserter(utf32result));
+assert (utf32result.size() == 2);
+
+

+ In case of an invalid UTF-8 seqence, a utf8::invalid_utf8 exception is + thrown. If end does not point to the past-of-end of a UTF-8 seqence, a + utf8::not_enough_room exception is thrown. +

+

+ utf8::find_invalid +

+

+ Available in version 1.0 and later. +

+

+ Detects an invalid sequence within a UTF-8 string. +

+
+template <typename octet_iterator> 
+octet_iterator find_invalid(octet_iterator start, octet_iterator end);
+
+

+ octet_iterator: an input iterator.
+ start: an iterator pointing to the beginning of the UTF-8 string to + test for validity.
+ end: an iterator pointing to pass-the-end of the UTF-8 string to test + for validity.
+ Return value: an iterator pointing to the first + invalid octet in the UTF-8 string. In case none were found, equals + end. +

+

+ Example of use: +

+
+char utf_invalid[] = "\xe6\x97\xa5\xd1\x88\xfa";
+char* invalid = find_invalid(utf_invalid, utf_invalid + 6);
+assert (invalid == utf_invalid + 5);
+
+

+ This function is typically used to make sure a UTF-8 string is valid before + processing it with other functions. It is especially important to call it if before + doing any of the unchecked operations on it. +

+

+ utf8::is_valid +

+

+ Available in version 1.0 and later. +

+

+ Checks whether a sequence of octets is a valid UTF-8 string. +

+
+template <typename octet_iterator> 
+bool is_valid(octet_iterator start, octet_iterator end);
+   
+
+

+ octet_iterator: an input iterator.
+ start: an iterator pointing to the beginning of the UTF-8 string to + test for validity.
+ end: an iterator pointing to pass-the-end of the UTF-8 string to test + for validity.
+ Return value: true if the sequence + is a valid UTF-8 string; false if not. +

+ Example of use: +
+char utf_invalid[] = "\xe6\x97\xa5\xd1\x88\xfa";
+bool bvalid = is_valid(utf_invalid, utf_invalid + 6);
+assert (bvalid == false);
+
+

+ is_valid is a shorthand for find_invalid(start, end) == + end;. You may want to use it to make sure that a byte seqence is a valid + UTF-8 string without the need to know where it fails if it is not valid. +

+

+ utf8::replace_invalid +

+

+ Available in version 2.0 and later. +

+

+ Replaces all invalid UTF-8 sequences within a string with a replacement marker. +

+
+template <typename octet_iterator, typename output_iterator>
+output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement);
+template <typename octet_iterator, typename output_iterator>
+output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out);
+   
+
+

+ octet_iterator: an input iterator.
+ output_iterator: an output iterator.
+ start: an iterator pointing to the beginning of the UTF-8 string to + look for invalid UTF-8 sequences.
+ end: an iterator pointing to pass-the-end of the UTF-8 string to look + for invalid UTF-8 sequences.
+ out: An output iterator to the range where the result of replacement + is stored.
+ replacement: A Unicode code point for the replacement marker. The + version without this parameter assumes the value 0xfffd
+ Return value: An iterator pointing to the place + after the UTF-8 string with replaced invalid sequences. +

+

+ Example of use: +

+
+char invalid_sequence[] = "a\x80\xe0\xa0\xc0\xaf\xed\xa0\x80z";
+vector<char> replace_invalid_result;
+replace_invalid (invalid_sequence, invalid_sequence + sizeof(invalid_sequence), back_inserter(replace_invalid_result), '?');
+bvalid = is_valid(replace_invalid_result.begin(), replace_invalid_result.end());
+assert (bvalid);
+char* fixed_invalid_sequence = "a????z";
+assert (std::equal(replace_invalid_result.begin(), replace_invalid_result.end(), fixed_invalid_sequence));
+
+

+ replace_invalid does not perform in-place replacement of invalid + sequences. Rather, it produces a copy of the original string with the invalid + sequences replaced with a replacement marker. Therefore, out must not + be in the [start, end] range. +

+

+ If end does not point to the past-of-end of a UTF-8 sequence, a + utf8::not_enough_room exception is thrown. +

+

+ utf8::starts_with_bom +

+

+ Available in version 2.3 and later. Relaces deprecated is_bom() function. +

+

+ Checks whether an octet sequence starts with a UTF-8 byte order mark (BOM) +

+
+template <typename octet_iterator> 
+bool starts_with_bom (octet_iterator it, octet_iterator end);
+
+

+ octet_iterator: an input iterator.
+ it: beginning of the octet sequence to check
+ end: pass-end of the sequence to check
+ Return value: true if the sequence + starts with a UTF-8 byte order mark; false if not. +

+

+ Example of use: +

+
+unsigned char byte_order_mark[] = {0xef, 0xbb, 0xbf};
+bool bbom = starts_with_bom(byte_order_mark, byte_order_mark + sizeof(byte_order_mark));
+assert (bbom == true);
+
+

+ The typical use of this function is to check the first three bytes of a file. If + they form the UTF-8 BOM, we want to skip them before processing the actual UTF-8 + encoded text. +

+

+ utf8::is_bom +

+

+ Available in version 1.0 and later. Deprecated in version 2.3. starts_with_bom() should be used + instead. +

+

+ Checks whether a sequence of three octets is a UTF-8 byte order mark (BOM) +

+
+template <typename octet_iterator> 
+bool is_bom (octet_iterator it);  // Deprecated
+
+

+ octet_iterator: an input iterator.
+ it: beginning of the 3-octet sequence to check
+ Return value: true if the sequence + is UTF-8 byte order mark; false if not. +

+

+ Example of use: +

+
+unsigned char byte_order_mark[] = {0xef, 0xbb, 0xbf};
+bool bbom = is_bom(byte_order_mark);
+assert (bbom == true);
+
+

+ The typical use of this function is to check the first three bytes of a file. If + they form the UTF-8 BOM, we want to skip them before processing the actual UTF-8 + encoded text. +

+

+ If a sequence is + shorter than three bytes, an invalid iterator will be dereferenced. Therefore, this function is deprecated + in favor of starts_with_bom()that takes the end of sequence as an argument. +

+

+ Types From utf8 Namespace +

+

utf8::exception +

+

+ Available in version 2.3 and later. +

+

+ Base class for the exceptions thrown by UTF CPP library functions. +

+
+class exception : public std::exception {};
+
+

+ Example of use: +

+
+try {
+  code_that_uses_utf_cpp_library();
+}
+catch(const utf8::exception& utfcpp_ex) {
+  cerr << utfcpp_ex.what();
+}
+
+ +

utf8::invalid_code_point +

+

+ Available in version 1.0 and later. +

+

+ Thrown by UTF8 CPP functions such as advance and next if an UTF-8 sequence represents and invalid code point. +

+ +
+class invalid_code_point : public exception {
+public: 
+    uint32_t code_point() const;
+};
+
+
+

+ Member function code_point() can be used to determine the invalid code point that + caused the exception to be thrown. +

+

utf8::invalid_utf8 +

+

+ Available in version 1.0 and later. +

+

+ Thrown by UTF8 CPP functions such as next and prior if an invalid UTF-8 sequence + is detected during decoding. +

+ +
+class invalid_utf8 : public exception {
+public: 
+    uint8_t utf8_octet() const;
+};
+
+ +

+ Member function utf8_octet() can be used to determine the beginning of the byte + sequence that caused the exception to be thrown. +

+ +

utf8::invalid_utf16 +

+

+ Available in version 1.0 and later. +

+

+ Thrown by UTF8 CPP function utf16to8 if an invalid UTF-16 sequence + is detected during decoding. +

+ +
+class invalid_utf16 : public exception {
+public: 
+    uint16_t utf16_word() const;
+};
+
+ +

+ Member function utf16_word() can be used to determine the UTF-16 code unit + that caused the exception to be thrown. +

+

utf8::not_enough_room +

+

+ Available in version 1.0 and later. +

+

+ Thrown by UTF8 CPP functions such as next if the end of the decoded UTF-8 sequence + was reached before the code point was decoded. +

+ +
+class not_enough_room : public exception {};
+
+

+ utf8::iterator +

+

+ Available in version 2.0 and later. +

+

+ Adapts the underlying octet iterator to iterate over the sequence of code points, + rather than raw octets. +

+
+template <typename octet_iterator>
+class iterator;
+
+ +
Member functions
+
+
iterator();
the deafult constructor; the underlying octet_iterator is + constructed with its default constructor. +
explicit iterator (const octet_iterator& octet_it, + const octet_iterator& range_start, + const octet_iterator& range_end);
a constructor + that initializes the underlying octet_iterator with octet_it + and sets the range in which the iterator is considered valid. +
octet_iterator base () const;
returns the + underlying octet_iterator. +
uint32_t operator * () const;
decodes the utf-8 sequence + the underlying octet_iterator is pointing to and returns the code point. +
bool operator == (const iterator& rhs) + const;
returns true + if the two underlaying iterators are equal. +
bool operator != (const iterator& rhs) + const;
returns true + if the two underlaying iterators are not equal. +
iterator& operator ++ ();
the prefix increment - moves + the iterator to the next UTF-8 encoded code point. +
iterator operator ++ (int);
+ the postfix increment - moves the iterator to the next UTF-8 encoded code point and returns the current one. +
iterator& operator -- ();
the prefix decrement - moves + the iterator to the previous UTF-8 encoded code point. +
iterator operator -- (int);
+ the postfix decrement - moves the iterator to the previous UTF-8 encoded code point and returns the current one. +
+

+ Example of use: +

+
+char* threechars = "\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
+utf8::iterator<char*> it(threechars, threechars, threechars + 9);
+utf8::iterator<char*> it2 = it;
+assert (it2 == it);
+assert (*it == 0x10346);
+assert (*(++it) == 0x65e5);
+assert ((*it++) == 0x65e5);
+assert (*it == 0x0448);
+assert (it != it2);
+utf8::iterator<char*> endit (threechars + 9, threechars, threechars + 9);  
+assert (++it == endit);
+assert (*(--it) == 0x0448);
+assert ((*it--) == 0x0448);
+assert (*it == 0x65e5);
+assert (--it == utf8::iterator<char*>(threechars, threechars, threechars + 9));
+assert (*it == 0x10346);
+
+

+ The purpose of utf8::iterator adapter is to enable easy iteration as well as the use of STL + algorithms with UTF-8 encoded strings. Increment and decrement operators are implemented in terms of + utf8::next() and utf8::prior() functions. +

+

+ Note that utf8::iterator adapter is a checked iterator. It operates on the range specified in + the constructor; any attempt to go out of that range will result in an exception. Even the comparison operators + require both iterator object to be constructed against the same range - otherwise an exception is thrown. Typically, + the range will be determined by sequence container functions begin and end, i.e.: +

+
+std::string s = "example";
+utf8::iterator i (s.begin(), s.begin(), s.end());
+
+

+ Functions From utf8::unchecked Namespace +

+

+ utf8::unchecked::append +

+

+ Available in version 1.0 and later. +

+

+ Encodes a 32 bit code point as a UTF-8 sequence of octets and appends the sequence + to a UTF-8 string. +

+
+template <typename octet_iterator>
+octet_iterator append(uint32_t cp, octet_iterator result);
+   
+
+

+ cp: A 32 bit integer representing a code point to append to the + sequence.
+ result: An output iterator to the place in the sequence where to + append the code point.
+ Return value: An iterator pointing to the place + after the newly appended sequence. +

+

+ Example of use: +

+
+unsigned char u[5] = {0,0,0,0,0};
+unsigned char* end = unchecked::append(0x0448, u);
+assert (u[0] == 0xd1 && u[1] == 0x88 && u[2] == 0 && u[3] == 0 && u[4] == 0);
+
+

+ This is a faster but less safe version of utf8::append. It does not + check for validity of the supplied code point, and may produce an invalid UTF-8 + sequence. +

+

+ utf8::unchecked::next +

+

+ Available in version 1.0 and later. +

+

+ Given the iterator to the beginning of a UTF-8 sequence, it returns the code point + and moves the iterator to the next position. +

+
+template <typename octet_iterator>
+uint32_t next(octet_iterator& it);
+   
+
+

+ it: a reference to an iterator pointing to the beginning of an UTF-8 + encoded code point. After the function returns, it is incremented to point to the + beginning of the next code point.
+ Return value: the 32 bit representation of the + processed UTF-8 code point. +

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+char* w = twochars;
+int cp = unchecked::next(w);
+assert (cp == 0x65e5);
+assert (w == twochars + 3);
+
+

+ This is a faster but less safe version of utf8::next. It does not + check for validity of the supplied UTF-8 sequence. +

+

+ utf8::unchecked::peek_next +

+

+ Available in version 2.1 and later. +

+

+ Given the iterator to the beginning of a UTF-8 sequence, it returns the code point. +

+
+template <typename octet_iterator>
+uint32_t peek_next(octet_iterator it);
+   
+
+

+ it: an iterator pointing to the beginning of an UTF-8 + encoded code point.
+ Return value: the 32 bit representation of the + processed UTF-8 code point. +

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+char* w = twochars;
+int cp = unchecked::peek_next(w);
+assert (cp == 0x65e5);
+assert (w == twochars);
+
+

+ This is a faster but less safe version of utf8::peek_next. It does not + check for validity of the supplied UTF-8 sequence. +

+

+ utf8::unchecked::prior +

+

+ Available in version 1.02 and later. +

+

+ Given a reference to an iterator pointing to an octet in a UTF-8 seqence, it + decreases the iterator until it hits the beginning of the previous UTF-8 encoded + code point and returns the 32 bits representation of the code point. +

+
+template <typename octet_iterator>
+uint32_t prior(octet_iterator& it);
+   
+
+

+ it: a reference pointing to an octet within a UTF-8 encoded string. + After the function returns, it is decremented to point to the beginning of the + previous code point.
+ Return value: the 32 bit representation of the + previous code point. +

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+char* w = twochars + 3;
+int cp = unchecked::prior (w);
+assert (cp == 0x65e5);
+assert (w == twochars);
+
+

+ This is a faster but less safe version of utf8::prior. It does not + check for validity of the supplied UTF-8 sequence and offers no boundary checking. +

+

+ utf8::unchecked::previous (deprecated, see utf8::unchecked::prior) +

+

+ Deprecated in version 1.02 and later. +

+

+ Given a reference to an iterator pointing to an octet in a UTF-8 seqence, it + decreases the iterator until it hits the beginning of the previous UTF-8 encoded + code point and returns the 32 bits representation of the code point. +

+
+template <typename octet_iterator>
+uint32_t previous(octet_iterator& it);
+   
+
+

+ it: a reference pointing to an octet within a UTF-8 encoded string. + After the function returns, it is decremented to point to the beginning of the + previous code point.
+ Return value: the 32 bit representation of the + previous code point. +

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+char* w = twochars + 3;
+int cp = unchecked::previous (w);
+assert (cp == 0x65e5);
+assert (w == twochars);
+
+

+ The reason this function is deprecated is just the consistency with the "checked" + versions, where prior should be used instead of previous. + In fact, unchecked::previous behaves exactly the same as + unchecked::prior +

+

+ This is a faster but less safe version of utf8::previous. It does not + check for validity of the supplied UTF-8 sequence and offers no boundary checking. +

+

+ utf8::unchecked::advance +

+

+ Available in version 1.0 and later. +

+

+ Advances an iterator by the specified number of code points within an UTF-8 + sequence. +

+
+template <typename octet_iterator, typename distance_type>
+void advance (octet_iterator& it, distance_type n);
+   
+
+

+ it: a reference to an iterator pointing to the beginning of an UTF-8 + encoded code point. After the function returns, it is incremented to point to the + nth following code point.
+ n: a positive integer that shows how many code points we want to + advance.
+

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+char* w = twochars;
+unchecked::advance (w, 2);
+assert (w == twochars + 5);
+
+

+ This function works only "forward". In case of a negative n, there is + no effect. +

+

+ This is a faster but less safe version of utf8::advance. It does not + check for validity of the supplied UTF-8 sequence and offers no boundary checking. +

+

+ utf8::unchecked::distance +

+

+ Available in version 1.0 and later. +

+

+ Given the iterators to two UTF-8 encoded code points in a seqence, returns the + number of code points between them. +

+
+template <typename octet_iterator>
+typename std::iterator_traits<octet_iterator>::difference_type distance (octet_iterator first, octet_iterator last);
+
+

+ first: an iterator to a beginning of a UTF-8 encoded code point.
+ last: an iterator to a "post-end" of the last UTF-8 encoded code + point in the sequence we are trying to determine the length. It can be the + beginning of a new code point, or not.
+ Return value the distance between the iterators, + in code points. +

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+size_t dist = utf8::unchecked::distance(twochars, twochars + 5);
+assert (dist == 2);
+
+

+ This is a faster but less safe version of utf8::distance. It does not + check for validity of the supplied UTF-8 sequence. +

+

+ utf8::unchecked::utf16to8 +

+

+ Available in version 1.0 and later. +

+

+ Converts a UTF-16 encoded string to UTF-8. +

+
+template <typename u16bit_iterator, typename octet_iterator>
+octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result);
+   
+
+

+ start: an iterator pointing to the beginning of the UTF-16 encoded + string to convert.
+ end: an iterator pointing to pass-the-end of the UTF-16 encoded + string to convert.
+ result: an output iterator to the place in the UTF-8 string where to + append the result of conversion.
+ Return value: An iterator pointing to the place + after the appended UTF-8 string. +

+

+ Example of use: +

+
+unsigned short utf16string[] = {0x41, 0x0448, 0x65e5, 0xd834, 0xdd1e};
+vector<unsigned char> utf8result;
+unchecked::utf16to8(utf16string, utf16string + 5, back_inserter(utf8result));
+assert (utf8result.size() == 10);    
+
+

+ This is a faster but less safe version of utf8::utf16to8. It does not + check for validity of the supplied UTF-16 sequence. +

+

+ utf8::unchecked::utf8to16 +

+

+ Available in version 1.0 and later. +

+

+ Converts an UTF-8 encoded string to UTF-16 +

+
+template <typename u16bit_iterator, typename octet_iterator>
+u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result);
+   
+
+

+ start: an iterator pointing to the beginning of the UTF-8 encoded + string to convert. < br /> end: an iterator pointing to + pass-the-end of the UTF-8 encoded string to convert.
+ result: an output iterator to the place in the UTF-16 string where to + append the result of conversion.
+ Return value: An iterator pointing to the place + after the appended UTF-16 string. +

+

+ Example of use: +

+
+char utf8_with_surrogates[] = "\xe6\x97\xa5\xd1\x88\xf0\x9d\x84\x9e";
+vector <unsigned short> utf16result;
+unchecked::utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result));
+assert (utf16result.size() == 4);
+assert (utf16result[2] == 0xd834);
+assert (utf16result[3] == 0xdd1e);
+
+

+ This is a faster but less safe version of utf8::utf8to16. It does not + check for validity of the supplied UTF-8 sequence. +

+

+ utf8::unchecked::utf32to8 +

+

+ Available in version 1.0 and later. +

+

+ Converts a UTF-32 encoded string to UTF-8. +

+
+template <typename octet_iterator, typename u32bit_iterator>
+octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result);
+   
+
+

+ start: an iterator pointing to the beginning of the UTF-32 encoded + string to convert.
+ end: an iterator pointing to pass-the-end of the UTF-32 encoded + string to convert.
+ result: an output iterator to the place in the UTF-8 string where to + append the result of conversion.
+ Return value: An iterator pointing to the place + after the appended UTF-8 string. +

+

+ Example of use: +

+
+int utf32string[] = {0x448, 0x65e5, 0x10346, 0};
+vector<unsigned char> utf8result;
+utf32to8(utf32string, utf32string + 3, back_inserter(utf8result));
+assert (utf8result.size() == 9);
+
+

+ This is a faster but less safe version of utf8::utf32to8. It does not + check for validity of the supplied UTF-32 sequence. +

+

+ utf8::unchecked::utf8to32 +

+

+ Available in version 1.0 and later. +

+

+ Converts a UTF-8 encoded string to UTF-32. +

+
+template <typename octet_iterator, typename u32bit_iterator>
+u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result);
+   
+
+

+ start: an iterator pointing to the beginning of the UTF-8 encoded + string to convert.
+ end: an iterator pointing to pass-the-end of the UTF-8 encoded string + to convert.
+ result: an output iterator to the place in the UTF-32 string where to + append the result of conversion.
+ Return value: An iterator pointing to the place + after the appended UTF-32 string. +

+

+ Example of use: +

+
+char* twochars = "\xe6\x97\xa5\xd1\x88";
+vector<int> utf32result;
+unchecked::utf8to32(twochars, twochars + 5, back_inserter(utf32result));
+assert (utf32result.size() == 2);
+
+

+ This is a faster but less safe version of utf8::utf8to32. It does not + check for validity of the supplied UTF-8 sequence. +

+

+ Types From utf8::unchecked Namespace +

+

+ utf8::iterator +

+

+ Available in version 2.0 and later. +

+

+ Adapts the underlying octet iterator to iterate over the sequence of code points, + rather than raw octets. +

+
+template <typename octet_iterator>
+class iterator;
+
+ +
Member functions
+
+
iterator();
the deafult constructor; the underlying octet_iterator is + constructed with its default constructor. +
explicit iterator (const octet_iterator& octet_it); +
a constructor + that initializes the underlying octet_iterator with octet_it +
octet_iterator base () const;
returns the + underlying octet_iterator. +
uint32_t operator * () const;
decodes the utf-8 sequence + the underlying octet_iterator is pointing to and returns the code point. +
bool operator == (const iterator& rhs) + const;
returns true + if the two underlaying iterators are equal. +
bool operator != (const iterator& rhs) + const;
returns true + if the two underlaying iterators are not equal. +
iterator& operator ++ ();
the prefix increment - moves + the iterator to the next UTF-8 encoded code point. +
iterator operator ++ (int);
+ the postfix increment - moves the iterator to the next UTF-8 encoded code point and returns the current one. +
iterator& operator -- ();
the prefix decrement - moves + the iterator to the previous UTF-8 encoded code point. +
iterator operator -- (int);
+ the postfix decrement - moves the iterator to the previous UTF-8 encoded code point and returns the current one. +
+

+ Example of use: +

+
+char* threechars = "\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
+utf8::unchecked::iterator<char*> un_it(threechars);
+utf8::unchecked::iterator<char*> un_it2 = un_it;
+assert (un_it2 == un_it);
+assert (*un_it == 0x10346);
+assert (*(++un_it) == 0x65e5);
+assert ((*un_it++) == 0x65e5);
+assert (*un_it == 0x0448);
+assert (un_it != un_it2);
+utf8::::unchecked::iterator<char*> un_endit (threechars + 9);  
+assert (++un_it == un_endit);
+assert (*(--un_it) == 0x0448);
+assert ((*un_it--) == 0x0448);
+assert (*un_it == 0x65e5);
+assert (--un_it == utf8::unchecked::iterator<char*>(threechars));
+assert (*un_it == 0x10346);
+
+

+ This is an unchecked version of utf8::iterator. It is faster in many cases, but offers + no validity or range checks. +

+

+ Points of interest +

+

+ Design goals and decisions +

+

+ The library was designed to be: +

+
    +
  1. + Generic: for better or worse, there are many C++ string classes out there, and + the library should work with as many of them as possible. +
    2. +
  2. + Portable: the library should be portable both accross different platforms and + compilers. The only non-portable code is a small section that declares unsigned + integers of different sizes: three typedefs. They can be changed by the users of + the library if they don't match their platform. The default setting should work + for Windows (both 32 and 64 bit), and most 32 bit and 64 bit Unix derivatives. +
    3. +
  3. + Lightweight: follow the "pay only for what you use" guideline. +
    4. +
  4. + Unintrusive: avoid forcing any particular design or even programming style on the + user. This is a library, not a framework. +
    5. +
+

+ Alternatives +

+

+ In case you want to look into other means of working with UTF-8 strings from C++, + here is the list of solutions I am aware of: +

+
    +
  1. + ICU Library. It is very powerful, + complete, feature-rich, mature, and widely used. Also big, intrusive, + non-generic, and doesn't play well with the Standard Library. I definitelly + recommend looking at ICU even if you don't plan to use it. +
    2. +
  2. + C++11 language and library features. Still far from complete, and not widely + supported by compiler vendors. +
    3. +
  3. + Glib::ustring. + A class specifically made to work with UTF-8 strings, and also feel like + std::string. If you prefer to have yet another string class in your + code, it may be worth a look. Be aware of the licensing issues, though. +
    4. +
  4. + Platform dependent solutions: Windows and POSIX have functions to convert strings + from one encoding to another. That is only a subset of what my library offers, + but if that is all you need it may be good enough. +
    5. +
+ +
    +
  1. + The Unicode Consortium. +
    2. +
  2. + ICU Library. +
    3. +
  3. + UTF-8 at Wikipedia +
    4. +
  4. + UTF-8 and Unicode FAQ for + Unix/Linux +
    5. +
+ + diff --git a/contrib/utf8cpp/source/utf8.h b/contrib/utf8cpp/source/utf8.h new file mode 100644 index 000000000..82b13f59f --- /dev/null +++ b/contrib/utf8cpp/source/utf8.h @@ -0,0 +1,34 @@ +// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include "utf8/checked.h" +#include "utf8/unchecked.h" + +#endif // header guard diff --git a/contrib/utf8cpp/source/utf8/checked.h b/contrib/utf8cpp/source/utf8/checked.h new file mode 100644 index 000000000..133115513 --- /dev/null +++ b/contrib/utf8cpp/source/utf8/checked.h @@ -0,0 +1,327 @@ +// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include "core.h" +#include + +namespace utf8 +{ + // Base for the exceptions that may be thrown from the library + class exception : public ::std::exception { + }; + + // Exceptions that may be thrown from the library functions. + class invalid_code_point : public exception { + uint32_t cp; + public: + invalid_code_point(uint32_t cp) : cp(cp) {} + virtual const char* what() const throw() { return "Invalid code point"; } + uint32_t code_point() const {return cp;} + }; + + class invalid_utf8 : public exception { + uint8_t u8; + public: + invalid_utf8 (uint8_t u) : u8(u) {} + virtual const char* what() const throw() { return "Invalid UTF-8"; } + uint8_t utf8_octet() const {return u8;} + }; + + class invalid_utf16 : public exception { + uint16_t u16; + public: + invalid_utf16 (uint16_t u) : u16(u) {} + virtual const char* what() const throw() { return "Invalid UTF-16"; } + uint16_t utf16_word() const {return u16;} + }; + + class not_enough_room : public exception { + public: + virtual const char* what() const throw() { return "Not enough space"; } + }; + + /// The library API - functions intended to be called by the users + + template + octet_iterator append(uint32_t cp, octet_iterator result) + { + if (!utf8::internal::is_code_point_valid(cp)) + throw invalid_code_point(cp); + + if (cp < 0x80) // one octet + *(result++) = static_cast(cp); + else if (cp < 0x800) { // two octets + *(result++) = static_cast((cp >> 6) | 0xc0); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp < 0x10000) { // three octets + *(result++) = static_cast((cp >> 12) | 0xe0); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else { // four octets + *(result++) = static_cast((cp >> 18) | 0xf0); + *(result++) = static_cast(((cp >> 12) & 0x3f) | 0x80); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + return result; + } + + template + output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement) + { + while (start != end) { + octet_iterator sequence_start = start; + internal::utf_error err_code = utf8::internal::validate_next(start, end); + switch (err_code) { + case internal::UTF8_OK : + for (octet_iterator it = sequence_start; it != start; ++it) + *out++ = *it; + break; + case internal::NOT_ENOUGH_ROOM: + throw not_enough_room(); + case internal::INVALID_LEAD: + out = utf8::append (replacement, out); + ++start; + break; + case internal::INCOMPLETE_SEQUENCE: + case internal::OVERLONG_SEQUENCE: + case internal::INVALID_CODE_POINT: + out = utf8::append (replacement, out); + ++start; + // just one replacement mark for the sequence + while (start != end && utf8::internal::is_trail(*start)) + ++start; + break; + } + } + return out; + } + + template + inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out) + { + static const uint32_t replacement_marker = utf8::internal::mask16(0xfffd); + return utf8::replace_invalid(start, end, out, replacement_marker); + } + + template + uint32_t next(octet_iterator& it, octet_iterator end) + { + uint32_t cp = 0; + internal::utf_error err_code = utf8::internal::validate_next(it, end, cp); + switch (err_code) { + case internal::UTF8_OK : + break; + case internal::NOT_ENOUGH_ROOM : + throw not_enough_room(); + case internal::INVALID_LEAD : + case internal::INCOMPLETE_SEQUENCE : + case internal::OVERLONG_SEQUENCE : + throw invalid_utf8(*it); + case internal::INVALID_CODE_POINT : + throw invalid_code_point(cp); + } + return cp; + } + + template + uint32_t peek_next(octet_iterator it, octet_iterator end) + { + return utf8::next(it, end); + } + + template + uint32_t prior(octet_iterator& it, octet_iterator start) + { + // can't do much if it == start + if (it == start) + throw not_enough_room(); + + octet_iterator end = it; + // Go back until we hit either a lead octet or start + while (utf8::internal::is_trail(*(--it))) + if (it == start) + throw invalid_utf8(*it); // error - no lead byte in the sequence + return utf8::peek_next(it, end); + } + + /// Deprecated in versions that include "prior" + template + uint32_t previous(octet_iterator& it, octet_iterator pass_start) + { + octet_iterator end = it; + while (utf8::internal::is_trail(*(--it))) + if (it == pass_start) + throw invalid_utf8(*it); // error - no lead byte in the sequence + octet_iterator temp = it; + return utf8::next(temp, end); + } + + template + void advance (octet_iterator& it, distance_type n, octet_iterator end) + { + for (distance_type i = 0; i < n; ++i) + utf8::next(it, end); + } + + template + typename std::iterator_traits::difference_type + distance (octet_iterator first, octet_iterator last) + { + typename std::iterator_traits::difference_type dist; + for (dist = 0; first < last; ++dist) + utf8::next(first, last); + return dist; + } + + template + octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) + { + while (start != end) { + uint32_t cp = utf8::internal::mask16(*start++); + // Take care of surrogate pairs first + if (utf8::internal::is_lead_surrogate(cp)) { + if (start != end) { + uint32_t trail_surrogate = utf8::internal::mask16(*start++); + if (utf8::internal::is_trail_surrogate(trail_surrogate)) + cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; + else + throw invalid_utf16(static_cast(trail_surrogate)); + } + else + throw invalid_utf16(static_cast(cp)); + + } + // Lone trail surrogate + else if (utf8::internal::is_trail_surrogate(cp)) + throw invalid_utf16(static_cast(cp)); + + result = utf8::append(cp, result); + } + return result; + } + + template + u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) + { + while (start != end) { + uint32_t cp = utf8::next(start, end); + if (cp > 0xffff) { //make a surrogate pair + *result++ = static_cast((cp >> 10) + internal::LEAD_OFFSET); + *result++ = static_cast((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); + } + else + *result++ = static_cast(cp); + } + return result; + } + + template + octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result) + { + while (start != end) + result = utf8::append(*(start++), result); + + return result; + } + + template + u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) + { + while (start != end) + (*result++) = utf8::next(start, end); + + return result; + } + + // The iterator class + template + class iterator : public std::iterator { + octet_iterator it; + octet_iterator range_start; + octet_iterator range_end; + public: + iterator () {} + explicit iterator (const octet_iterator& octet_it, + const octet_iterator& range_start, + const octet_iterator& range_end) : + it(octet_it), range_start(range_start), range_end(range_end) + { + if (it < range_start || it > range_end) + throw std::out_of_range("Invalid utf-8 iterator position"); + } + // the default "big three" are OK + octet_iterator base () const { return it; } + uint32_t operator * () const + { + octet_iterator temp = it; + return utf8::next(temp, range_end); + } + bool operator == (const iterator& rhs) const + { + if (range_start != rhs.range_start || range_end != rhs.range_end) + throw std::logic_error("Comparing utf-8 iterators defined with different ranges"); + return (it == rhs.it); + } + bool operator != (const iterator& rhs) const + { + return !(operator == (rhs)); + } + iterator& operator ++ () + { + utf8::next(it, range_end); + return *this; + } + iterator operator ++ (int) + { + iterator temp = *this; + utf8::next(it, range_end); + return temp; + } + iterator& operator -- () + { + utf8::prior(it, range_start); + return *this; + } + iterator operator -- (int) + { + iterator temp = *this; + utf8::prior(it, range_start); + return temp; + } + }; // class iterator + +} // namespace utf8 + +#endif //header guard + + diff --git a/contrib/utf8cpp/source/utf8/core.h b/contrib/utf8cpp/source/utf8/core.h new file mode 100644 index 000000000..693d388c0 --- /dev/null +++ b/contrib/utf8cpp/source/utf8/core.h @@ -0,0 +1,329 @@ +// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include + +namespace utf8 +{ + // The typedefs for 8-bit, 16-bit and 32-bit unsigned integers + // You may need to change them to match your system. + // These typedefs have the same names as ones from cstdint, or boost/cstdint + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + +// Helper code - not intended to be directly called by the library users. May be changed at any time +namespace internal +{ + // Unicode constants + // Leading (high) surrogates: 0xd800 - 0xdbff + // Trailing (low) surrogates: 0xdc00 - 0xdfff + const uint16_t LEAD_SURROGATE_MIN = 0xd800u; + const uint16_t LEAD_SURROGATE_MAX = 0xdbffu; + const uint16_t TRAIL_SURROGATE_MIN = 0xdc00u; + const uint16_t TRAIL_SURROGATE_MAX = 0xdfffu; + const uint16_t LEAD_OFFSET = LEAD_SURROGATE_MIN - (0x10000 >> 10); + const uint32_t SURROGATE_OFFSET = 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN; + + // Maximum valid value for a Unicode code point + const uint32_t CODE_POINT_MAX = 0x0010ffffu; + + template + inline uint8_t mask8(octet_type oc) + { + return static_cast(0xff & oc); + } + template + inline uint16_t mask16(u16_type oc) + { + return static_cast(0xffff & oc); + } + template + inline bool is_trail(octet_type oc) + { + return ((utf8::internal::mask8(oc) >> 6) == 0x2); + } + + template + inline bool is_lead_surrogate(u16 cp) + { + return (cp >= LEAD_SURROGATE_MIN && cp <= LEAD_SURROGATE_MAX); + } + + template + inline bool is_trail_surrogate(u16 cp) + { + return (cp >= TRAIL_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); + } + + template + inline bool is_surrogate(u16 cp) + { + return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); + } + + template + inline bool is_code_point_valid(u32 cp) + { + return (cp <= CODE_POINT_MAX && !utf8::internal::is_surrogate(cp)); + } + + template + inline typename std::iterator_traits::difference_type + sequence_length(octet_iterator lead_it) + { + uint8_t lead = utf8::internal::mask8(*lead_it); + if (lead < 0x80) + return 1; + else if ((lead >> 5) == 0x6) + return 2; + else if ((lead >> 4) == 0xe) + return 3; + else if ((lead >> 3) == 0x1e) + return 4; + else + return 0; + } + + template + inline bool is_overlong_sequence(uint32_t cp, octet_difference_type length) + { + if (cp < 0x80) { + if (length != 1) + return true; + } + else if (cp < 0x800) { + if (length != 2) + return true; + } + else if (cp < 0x10000) { + if (length != 3) + return true; + } + + return false; + } + + enum utf_error {UTF8_OK, NOT_ENOUGH_ROOM, INVALID_LEAD, INCOMPLETE_SEQUENCE, OVERLONG_SEQUENCE, INVALID_CODE_POINT}; + + /// Helper for get_sequence_x + template + utf_error increase_safely(octet_iterator& it, octet_iterator end) + { + if (++it == end) + return NOT_ENOUGH_ROOM; + + if (!utf8::internal::is_trail(*it)) + return INCOMPLETE_SEQUENCE; + + return UTF8_OK; + } + + #define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END) {utf_error ret = increase_safely(IT, END); if (ret != UTF8_OK) return ret;} + + /// get_sequence_x functions decode utf-8 sequences of the length x + template + utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + return UTF8_OK; + } + + template + utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point = ((code_point << 6) & 0x7ff) + ((*it) & 0x3f); + + return UTF8_OK; + } + + template + utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point = ((code_point << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point += (*it) & 0x3f; + + return UTF8_OK; + } + + template + utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point = ((code_point << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point += (utf8::internal::mask8(*it) << 6) & 0xfff; + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point += (*it) & 0x3f; + + return UTF8_OK; + } + + #undef UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR + + template + utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + // Save the original value of it so we can go back in case of failure + // Of course, it does not make much sense with i.e. stream iterators + octet_iterator original_it = it; + + uint32_t cp = 0; + // Determine the sequence length based on the lead octet + typedef typename std::iterator_traits::difference_type octet_difference_type; + const octet_difference_type length = utf8::internal::sequence_length(it); + + // Get trail octets and calculate the code point + utf_error err = UTF8_OK; + switch (length) { + case 0: + return INVALID_LEAD; + case 1: + err = utf8::internal::get_sequence_1(it, end, cp); + break; + case 2: + err = utf8::internal::get_sequence_2(it, end, cp); + break; + case 3: + err = utf8::internal::get_sequence_3(it, end, cp); + break; + case 4: + err = utf8::internal::get_sequence_4(it, end, cp); + break; + } + + if (err == UTF8_OK) { + // Decoding succeeded. Now, security checks... + if (utf8::internal::is_code_point_valid(cp)) { + if (!utf8::internal::is_overlong_sequence(cp, length)){ + // Passed! Return here. + code_point = cp; + ++it; + return UTF8_OK; + } + else + err = OVERLONG_SEQUENCE; + } + else + err = INVALID_CODE_POINT; + } + + // Failure branch - restore the original value of the iterator + it = original_it; + return err; + } + + template + inline utf_error validate_next(octet_iterator& it, octet_iterator end) { + uint32_t ignored; + return utf8::internal::validate_next(it, end, ignored); + } + +} // namespace internal + + /// The library API - functions intended to be called by the users + + // Byte order mark + const uint8_t bom[] = {0xef, 0xbb, 0xbf}; + + template + octet_iterator find_invalid(octet_iterator start, octet_iterator end) + { + octet_iterator result = start; + while (result != end) { + utf8::internal::utf_error err_code = utf8::internal::validate_next(result, end); + if (err_code != internal::UTF8_OK) + return result; + } + return result; + } + + template + inline bool is_valid(octet_iterator start, octet_iterator end) + { + return (utf8::find_invalid(start, end) == end); + } + + template + inline bool starts_with_bom (octet_iterator it, octet_iterator end) + { + return ( + ((it != end) && (utf8::internal::mask8(*it++)) == bom[0]) && + ((it != end) && (utf8::internal::mask8(*it++)) == bom[1]) && + ((it != end) && (utf8::internal::mask8(*it)) == bom[2]) + ); + } + + //Deprecated in release 2.3 + template + inline bool is_bom (octet_iterator it) + { + return ( + (utf8::internal::mask8(*it++)) == bom[0] && + (utf8::internal::mask8(*it++)) == bom[1] && + (utf8::internal::mask8(*it)) == bom[2] + ); + } +} // namespace utf8 + +#endif // header guard + + diff --git a/contrib/utf8cpp/source/utf8/unchecked.h b/contrib/utf8cpp/source/utf8/unchecked.h new file mode 100644 index 000000000..cb2427166 --- /dev/null +++ b/contrib/utf8cpp/source/utf8/unchecked.h @@ -0,0 +1,228 @@ +// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include "core.h" + +namespace utf8 +{ + namespace unchecked + { + template + octet_iterator append(uint32_t cp, octet_iterator result) + { + if (cp < 0x80) // one octet + *(result++) = static_cast(cp); + else if (cp < 0x800) { // two octets + *(result++) = static_cast((cp >> 6) | 0xc0); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp < 0x10000) { // three octets + *(result++) = static_cast((cp >> 12) | 0xe0); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else { // four octets + *(result++) = static_cast((cp >> 18) | 0xf0); + *(result++) = static_cast(((cp >> 12) & 0x3f)| 0x80); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + return result; + } + + template + uint32_t next(octet_iterator& it) + { + uint32_t cp = utf8::internal::mask8(*it); + typename std::iterator_traits::difference_type length = utf8::internal::sequence_length(it); + switch (length) { + case 1: + break; + case 2: + it++; + cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f); + break; + case 3: + ++it; + cp = ((cp << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff); + ++it; + cp += (*it) & 0x3f; + break; + case 4: + ++it; + cp = ((cp << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff); + ++it; + cp += (utf8::internal::mask8(*it) << 6) & 0xfff; + ++it; + cp += (*it) & 0x3f; + break; + } + ++it; + return cp; + } + + template + uint32_t peek_next(octet_iterator it) + { + return utf8::unchecked::next(it); + } + + template + uint32_t prior(octet_iterator& it) + { + while (utf8::internal::is_trail(*(--it))) ; + octet_iterator temp = it; + return utf8::unchecked::next(temp); + } + + // Deprecated in versions that include prior, but only for the sake of consistency (see utf8::previous) + template + inline uint32_t previous(octet_iterator& it) + { + return utf8::unchecked::prior(it); + } + + template + void advance (octet_iterator& it, distance_type n) + { + for (distance_type i = 0; i < n; ++i) + utf8::unchecked::next(it); + } + + template + typename std::iterator_traits::difference_type + distance (octet_iterator first, octet_iterator last) + { + typename std::iterator_traits::difference_type dist; + for (dist = 0; first < last; ++dist) + utf8::unchecked::next(first); + return dist; + } + + template + octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) + { + while (start != end) { + uint32_t cp = utf8::internal::mask16(*start++); + // Take care of surrogate pairs first + if (utf8::internal::is_lead_surrogate(cp)) { + uint32_t trail_surrogate = utf8::internal::mask16(*start++); + cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; + } + result = utf8::unchecked::append(cp, result); + } + return result; + } + + template + u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) + { + while (start < end) { + uint32_t cp = utf8::unchecked::next(start); + if (cp > 0xffff) { //make a surrogate pair + *result++ = static_cast((cp >> 10) + internal::LEAD_OFFSET); + *result++ = static_cast((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); + } + else + *result++ = static_cast(cp); + } + return result; + } + + template + octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result) + { + while (start != end) + result = utf8::unchecked::append(*(start++), result); + + return result; + } + + template + u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) + { + while (start < end) + (*result++) = utf8::unchecked::next(start); + + return result; + } + + // The iterator class + template + class iterator : public std::iterator { + octet_iterator it; + public: + iterator () {} + explicit iterator (const octet_iterator& octet_it): it(octet_it) {} + // the default "big three" are OK + octet_iterator base () const { return it; } + uint32_t operator * () const + { + octet_iterator temp = it; + return utf8::unchecked::next(temp); + } + bool operator == (const iterator& rhs) const + { + return (it == rhs.it); + } + bool operator != (const iterator& rhs) const + { + return !(operator == (rhs)); + } + iterator& operator ++ () + { + ::std::advance(it, utf8::internal::sequence_length(it)); + return *this; + } + iterator operator ++ (int) + { + iterator temp = *this; + ::std::advance(it, utf8::internal::sequence_length(it)); + return temp; + } + iterator& operator -- () + { + utf8::unchecked::prior(it); + return *this; + } + iterator operator -- (int) + { + iterator temp = *this; + utf8::unchecked::prior(it); + return temp; + } + }; // class iterator + + } // namespace utf8::unchecked +} // namespace utf8 + + +#endif // header guard + From d7e13e67c4015b81492272b6399238db2da2441b Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Sat, 17 Jun 2017 19:38:47 +0200 Subject: [PATCH 2/3] Utf8: integrate new lib. --- code/BaseImporter.cpp | 1 - code/BaseImporter.h | 6 +- code/CMakeLists.txt | 6 - code/MMDPmxParser.cpp | 12 +- code/STEPFileEncoding.cpp | 59 ++-- contrib/ConvertUTF/ConvertUTF.c | 539 -------------------------------- contrib/ConvertUTF/ConvertUTF.h | 151 --------- contrib/ConvertUTF/readme.txt | 43 --- 8 files changed, 39 insertions(+), 778 deletions(-) delete mode 100644 contrib/ConvertUTF/ConvertUTF.c delete mode 100644 contrib/ConvertUTF/ConvertUTF.h delete mode 100644 contrib/ConvertUTF/readme.txt diff --git a/code/BaseImporter.cpp b/code/BaseImporter.cpp index ba5faed80..9a57f860d 100644 --- a/code/BaseImporter.cpp +++ b/code/BaseImporter.cpp @@ -303,7 +303,6 @@ void BaseImporter::GetExtensionList(std::set& extensions) return false; } -//#include "../contrib/ConvertUTF/ConvertUTF.h" #include "../contrib/utf8cpp/source/utf8.h" // ------------------------------------------------------------------------------------------------ diff --git a/code/BaseImporter.h b/code/BaseImporter.h index 06ed0691f..cee54c1ad 100644 --- a/code/BaseImporter.h +++ b/code/BaseImporter.h @@ -61,7 +61,6 @@ class BaseProcess; class SharedPostProcessInfo; class IOStream; - // utility to do char4 to uint32 in a portable manner #define AI_MAKE_MAGIC(string) ((uint32_t)((string[0] << 24) + \ (string[1] << 16) + (string[2] << 8) + string[3])) @@ -194,14 +193,11 @@ public: const Importer* pImp ); - // ------------------------------------------------------------------- /** Called by #Importer::GetImporterInfo to get a description of * some loader features. Importers must provide this information. */ virtual const aiImporterDesc* GetInfo() const = 0; - - // ------------------------------------------------------------------- /** Called by #Importer::GetExtensionList for each loaded importer. * Take the extension list contained in the structure returned by @@ -317,7 +313,7 @@ public: // static utilities * @param Size of one token, in bytes. Maximally 16 bytes. * @return true if one of the given tokens was found * - * @note For convinence, the check is also performed for the + * @note For convenience, the check is also performed for the * byte-swapped variant of all tokens (big endian). Only for * tokens of size 2,4. */ diff --git a/code/CMakeLists.txt b/code/CMakeLists.txt index de515ba94..017420d5d 100644 --- a/code/CMakeLists.txt +++ b/code/CMakeLists.txt @@ -695,11 +695,6 @@ SET( Extra_SRCS ) SOURCE_GROUP( Extra FILES ${Extra_SRCS}) -SET( ConvertUTF_SRCS - ../contrib/ConvertUTF/ConvertUTF.h - ../contrib/ConvertUTF/ConvertUTF.c -) -SOURCE_GROUP( ConvertUTF FILES ${ConvertUTF_SRCS}) SET( Clipper_SRCS ../contrib/clipper/clipper.hpp @@ -835,7 +830,6 @@ SET( assimp_src # Third-party libraries ${IrrXML_SRCS} - ${ConvertUTF_SRCS} ${unzip_compile_SRCS} ${Poly2Tri_SRCS} ${Clipper_SRCS} diff --git a/code/MMDPmxParser.cpp b/code/MMDPmxParser.cpp index 352259df5..89a4ea6fc 100644 --- a/code/MMDPmxParser.cpp +++ b/code/MMDPmxParser.cpp @@ -1,7 +1,7 @@ #include #include "MMDPmxParser.h" +#include "../contrib/utf8cpp/source/utf8.h" #include "Exceptional.h" -#include "../contrib/ConvertUTF/ConvertUTF.h" namespace pmx { @@ -60,15 +60,17 @@ namespace pmx const unsigned int targetSize = size * 3; // enough to encode char* targetStart = new char[targetSize](); const char* targetReserved = targetStart; - ConversionFlags flags = ConversionFlags::lenientConversion; +// ConversionFlags flags = ConversionFlags::lenientConversion; - ConversionResult conversionResult; - if( ( conversionResult = ConvertUTF16toUTF8( + //ConversionResult conversionResult; + /*if( ( conversionResult = ConvertUTF16toUTF8( (const UTF16**)&sourceStart, (const UTF16*)(sourceStart + size), (UTF8**)&targetStart, (UTF8*)(targetStart + targetSize), flags) ) != ConversionResult::conversionOK) { throw DeadlyImportError( "Convert " + std::string(sourceStart) + " to UTF8 is not valid." ); - } + }*/ + + utf8::utf16to8( sourceStart, sourceStart + size, targetStart ); result.assign(targetReserved, targetStart - targetReserved); delete[] targetReserved; diff --git a/code/STEPFileEncoding.cpp b/code/STEPFileEncoding.cpp index 0bf868d1c..bb0cb646a 100644 --- a/code/STEPFileEncoding.cpp +++ b/code/STEPFileEncoding.cpp @@ -44,14 +44,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "STEPFileEncoding.h" #include "fast_atof.h" +#include "../contrib/utf8cpp/source/utf8.h" -#include "../contrib/ConvertUTF/ConvertUTF.h" +//#include "../contrib/ConvertUTF/ConvertUTF.h" #include using namespace Assimp; // roman1 to utf16 table -static const UTF16 mac_codetable[] = { +static const uint16_t mac_codetable[] = { // 0x20 unassig./nonprint. slots 0x0020 , 0x0021 , @@ -309,14 +310,15 @@ bool STEP::StringToUTF8(std::string& s) ai_assert(sizeof(mac_codetable) / sizeof(mac_codetable[0]) == 0x100-0x20); - const UTF32 unival = mac_codetable[macval - 0x20], *univalp = &unival; + const uint32_t unival = mac_codetable[macval - 0x20], *univalp = &unival; - UTF8 temp[5], *tempp = temp; - ai_assert(sizeof(UTF8) == 1); + unsigned char temp[5], *tempp = temp; + ai_assert(sizeof( unsigned char ) == 1); - if(ConvertUTF32toUTF8(&univalp, univalp+1, &tempp, tempp+sizeof(temp), lenientConversion) != conversionOK) { + utf8::utf32to8( univalp, univalp + 1, tempp ); + /*if(ConvertUTF32toUTF8(&univalp, univalp+1, &tempp, tempp+sizeof(temp), lenientConversion) != conversionOK) { return false; - } + }*/ const size_t outcount = static_cast(tempp-temp); @@ -355,28 +357,29 @@ bool STEP::StringToUTF8(std::string& s) } const size_t count = (j-basei)/4; - std::unique_ptr src(new UTF16[count]); + std::unique_ptr src(new uint16_t[count]); const char* cur = s.c_str() + basei; for (size_t k = 0; k < count; ++k, cur += 4) { - src[k] = (static_cast(HexOctetToDecimal(cur)) << 8u) | - static_cast(HexOctetToDecimal(cur+2)); + src[k] = (static_cast(HexOctetToDecimal(cur)) << 8u) | + static_cast(HexOctetToDecimal(cur+2)); } const size_t dcount = count * 3; // this is enough to hold all possible outputs - std::unique_ptr dest(new UTF8[dcount]); + std::unique_ptr dest(new unsigned char[dcount]); - const UTF16* srct = src.get(); - UTF8* destt = dest.get(); - if(ConvertUTF16toUTF8(&srct, srct+count, &destt, destt+dcount, lenientConversion) != conversionOK) { + const uint16_t* srct = src.get(); + unsigned char* destt = dest.get(); + utf8::utf16to8( srct, srct + count, destt ); + /*if(ConvertUTF16toUTF8(&srct, srct+count, &destt, destt+dcount, lenientConversion) != conversionOK) { return false; - } + }*/ const size_t outcount = static_cast(destt-dest.get()); s.erase(i,(j+4-i)); - ai_assert(sizeof(UTF8) == 1); + ai_assert(sizeof(unsigned char) == 1); s.insert(i, reinterpret_cast(dest.get()), outcount); i += outcount; @@ -388,37 +391,37 @@ bool STEP::StringToUTF8(std::string& s) } const size_t count = (j-basei)/8; - std::unique_ptr src(new UTF32[count]); + std::unique_ptr src(new uint32_t[count]); const char* cur = s.c_str() + basei; for (size_t k = 0; k < count; ++k, cur += 8) { - src[k] = (static_cast(HexOctetToDecimal(cur )) << 24u) | - (static_cast(HexOctetToDecimal(cur+2)) << 16u) | - (static_cast(HexOctetToDecimal(cur+4)) << 8u) | - (static_cast(HexOctetToDecimal(cur+6))); + src[k] = (static_cast(HexOctetToDecimal(cur )) << 24u) | + (static_cast(HexOctetToDecimal(cur+2)) << 16u) | + (static_cast(HexOctetToDecimal(cur+4)) << 8u) | + (static_cast(HexOctetToDecimal(cur+6))); } const size_t dcount = count * 5; // this is enough to hold all possible outputs - std::unique_ptr dest(new UTF8[dcount]); + std::unique_ptr dest(new unsigned char[dcount]); - const UTF32* srct = src.get(); - UTF8* destt = dest.get(); - if(ConvertUTF32toUTF8(&srct, srct+count, &destt, destt+dcount, lenientConversion) != conversionOK) { + const uint32_t* srct = src.get(); + unsigned char* destt = dest.get(); + utf8::utf32to8( srct, srct + count, destt ); + /*if(ConvertUTF32toUTF8(&srct, srct+count, &destt, destt+dcount, lenientConversion) != conversionOK) { return false; - } + }*/ const size_t outcount = static_cast(destt-dest.get()); s.erase(i,(j+4-i)); - ai_assert(sizeof(UTF8) == 1); + ai_assert(sizeof(unsigned char) == 1); s.insert(i, reinterpret_cast(dest.get()), outcount); i += outcount; continue; } } - break; // TODO: other encoding patterns? diff --git a/contrib/ConvertUTF/ConvertUTF.c b/contrib/ConvertUTF/ConvertUTF.c deleted file mode 100644 index 9b3deebd6..000000000 --- a/contrib/ConvertUTF/ConvertUTF.c +++ /dev/null @@ -1,539 +0,0 @@ -/* - * Copyright 2001-2004 Unicode, Inc. - * - * Disclaimer - * - * This source code is provided as is by Unicode, Inc. No claims are - * made as to fitness for any particular purpose. No warranties of any - * kind are expressed or implied. The recipient agrees to determine - * applicability of information provided. If this file has been - * purchased on magnetic or optical media from Unicode, Inc., the - * sole remedy for any claim will be exchange of defective media - * within 90 days of receipt. - * - * Limitations on Rights to Redistribute This Code - * - * Unicode, Inc. hereby grants the right to freely use the information - * supplied in this file in the creation of products supporting the - * Unicode Standard, and to make copies of this file in any form - * for internal or external distribution as long as this notice - * remains attached. - */ - -/* --------------------------------------------------------------------- - - Conversions between UTF32, UTF-16, and UTF-8. Source code file. - Author: Mark E. Davis, 1994. - Rev History: Rick McGowan, fixes & updates May 2001. - Sept 2001: fixed const & error conditions per - mods suggested by S. Parent & A. Lillich. - June 2002: Tim Dodd added detection and handling of incomplete - source sequences, enhanced error detection, added casts - to eliminate compiler warnings. - July 2003: slight mods to back out aggressive FFFE detection. - Jan 2004: updated switches in from-UTF8 conversions. - Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions. - - See the header file "ConvertUTF.h" for complete documentation. - ------------------------------------------------------------------------- */ - - -#include "ConvertUTF.h" -#ifdef CVTUTF_DEBUG -#include -#endif - -static const int halfShift = 10; /* used for shifting by 10 bits */ - -static const UTF32 halfBase = 0x0010000UL; -static const UTF32 halfMask = 0x3FFUL; - -#define UNI_SUR_HIGH_START (UTF32)0xD800 -#define UNI_SUR_HIGH_END (UTF32)0xDBFF -#define UNI_SUR_LOW_START (UTF32)0xDC00 -#define UNI_SUR_LOW_END (UTF32)0xDFFF -#define false 0 -#define true 1 - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF32toUTF16 ( - const UTF32** sourceStart, const UTF32* sourceEnd, - UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF32* source = *sourceStart; - UTF16* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch; - if (target >= targetEnd) { - result = targetExhausted; break; - } - ch = *source++; - if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */ - /* UTF-16 surrogate values are illegal in UTF-32; 0xffff or 0xfffe are both reserved values */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { - if (flags == strictConversion) { - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } else { - *target++ = (UTF16)ch; /* normal case */ - } - } else if (ch > UNI_MAX_LEGAL_UTF32) { - if (flags == strictConversion) { - result = sourceIllegal; - } else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } else { - /* target is a character in range 0xFFFF - 0x10FFFF. */ - if (target + 1 >= targetEnd) { - --source; /* Back up source pointer! */ - result = targetExhausted; break; - } - ch -= halfBase; - *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START); - *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START); - } - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF16toUTF32 ( - const UTF16** sourceStart, const UTF16* sourceEnd, - UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF16* source = *sourceStart; - UTF32* target = *targetStart; - UTF32 ch, ch2; - while (source < sourceEnd) { - const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */ - ch = *source++; - /* If we have a surrogate pair, convert to UTF32 first. */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) { - /* If the 16 bits following the high surrogate are in the source buffer... */ - if (source < sourceEnd) { - ch2 = *source; - /* If it's a low surrogate, convert to UTF32. */ - if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) { - ch = ((ch - UNI_SUR_HIGH_START) << halfShift) - + (ch2 - UNI_SUR_LOW_START) + halfBase; - ++source; - } else if (flags == strictConversion) { /* it's an unpaired high surrogate */ - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } else { /* We don't have the 16 bits following the high surrogate. */ - --source; /* return to the high surrogate */ - result = sourceExhausted; - break; - } - } else if (flags == strictConversion) { - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) { - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } - if (target >= targetEnd) { - source = oldSource; /* Back up source pointer! */ - result = targetExhausted; break; - } - *target++ = ch; - } - *sourceStart = source; - *targetStart = target; -#ifdef CVTUTF_DEBUG -if (result == sourceIllegal) { - fprintf(stderr, "ConvertUTF16toUTF32 illegal seq 0x%04x,%04x\n", ch, ch2); - fflush(stderr); -} -#endif - return result; -} - -/* --------------------------------------------------------------------- */ - -/* - * Index into the table below with the first byte of a UTF-8 sequence to - * get the number of trailing bytes that are supposed to follow it. - * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is - * left as-is for anyone who may want to do such conversion, which was - * allowed in earlier algorithms. - */ -static const char trailingBytesForUTF8[256] = { - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 -}; - -/* - * Magic values subtracted from a buffer value during UTF8 conversion. - * This table contains as many values as there might be trailing bytes - * in a UTF-8 sequence. - */ -static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, - 0x03C82080UL, 0xFA082080UL, 0x82082080UL }; - -/* - * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed - * into the first byte, depending on how many bytes follow. There are - * as many entries in this table as there are UTF-8 sequence types. - * (I.e., one byte sequence, two byte... etc.). Remember that sequencs - * for *legal* UTF-8 will be 4 or fewer bytes total. - */ -static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; - -/* --------------------------------------------------------------------- */ - -/* The interface converts a whole buffer to avoid function-call overhead. - * Constants have been gathered. Loops & conditionals have been removed as - * much as possible for efficiency, in favor of drop-through switches. - * (See "Note A" at the bottom of the file for equivalent code.) - * If your compiler supports it, the "isLegalUTF8" call can be turned - * into an inline function. - */ - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF16toUTF8 ( - const UTF16** sourceStart, const UTF16* sourceEnd, - UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF16* source = *sourceStart; - UTF8* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch; - unsigned short bytesToWrite = 0; - const UTF32 byteMask = 0xBF; - const UTF32 byteMark = 0x80; - const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */ - ch = *source++; - /* If we have a surrogate pair, convert to UTF32 first. */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) { - /* If the 16 bits following the high surrogate are in the source buffer... */ - if (source < sourceEnd) { - UTF32 ch2 = *source; - /* If it's a low surrogate, convert to UTF32. */ - if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) { - ch = ((ch - UNI_SUR_HIGH_START) << halfShift) - + (ch2 - UNI_SUR_LOW_START) + halfBase; - ++source; - } else if (flags == strictConversion) { /* it's an unpaired high surrogate */ - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } else { /* We don't have the 16 bits following the high surrogate. */ - --source; /* return to the high surrogate */ - result = sourceExhausted; - break; - } - } else if (flags == strictConversion) { - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) { - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } - /* Figure out how many bytes the result will require */ - if (ch < (UTF32)0x80) { bytesToWrite = 1; - } else if (ch < (UTF32)0x800) { bytesToWrite = 2; - } else if (ch < (UTF32)0x10000) { bytesToWrite = 3; - } else if (ch < (UTF32)0x110000) { bytesToWrite = 4; - } else { bytesToWrite = 3; - ch = UNI_REPLACEMENT_CHAR; - } - - target += bytesToWrite; - if (target > targetEnd) { - source = oldSource; /* Back up source pointer! */ - target -= bytesToWrite; result = targetExhausted; break; - } - switch (bytesToWrite) { /* note: everything falls through. */ - case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 1: *--target = (UTF8)(ch | firstByteMark[bytesToWrite]); - } - target += bytesToWrite; - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- */ - -/* - * Utility routine to tell whether a sequence of bytes is legal UTF-8. - * This must be called with the length pre-determined by the first byte. - * If not calling this from ConvertUTF8to*, then the length can be set by: - * length = trailingBytesForUTF8[*source]+1; - * and the sequence is illegal right away if there aren't that many bytes - * available. - * If presented with a length > 4, this returns false. The Unicode - * definition of UTF-8 goes up to 4-byte sequences. - */ - -static Boolean isLegalUTF8(const UTF8 *source, int length) { - UTF8 a; - const UTF8 *srcptr = source+length; - switch (length) { - default: return false; - /* Everything else falls through when "true"... */ - case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; - case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; - case 2: if ((a = (*--srcptr)) > 0xBF) return false; - - switch (*source) { - /* no fall-through in this inner switch */ - case 0xE0: if (a < 0xA0) return false; break; - case 0xED: if (a > 0x9F) return false; break; - case 0xF0: if (a < 0x90) return false; break; - case 0xF4: if (a > 0x8F) return false; break; - default: if (a < 0x80) return false; - } - - case 1: if (*source >= 0x80 && *source < 0xC2) return false; - } - if (*source > 0xF4) return false; - return true; -} - -/* --------------------------------------------------------------------- */ - -/* - * Exported function to return whether a UTF-8 sequence is legal or not. - * This is not used here; it's just exported. - */ -Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) { - int length = trailingBytesForUTF8[*source]+1; - if (source+length > sourceEnd) { - return false; - } - return isLegalUTF8(source, length); -} - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF8toUTF16 ( - const UTF8** sourceStart, const UTF8* sourceEnd, - UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF8* source = *sourceStart; - UTF16* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch = 0; - unsigned short extraBytesToRead = trailingBytesForUTF8[*source]; - if (source + extraBytesToRead >= sourceEnd) { - result = sourceExhausted; break; - } - /* Do this check whether lenient or strict */ - if (! isLegalUTF8(source, extraBytesToRead+1)) { - result = sourceIllegal; - break; - } - /* - * The cases all fall through. See "Note A" below. - */ - switch (extraBytesToRead) { - case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */ - case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */ - case 3: ch += *source++; ch <<= 6; - case 2: ch += *source++; ch <<= 6; - case 1: ch += *source++; ch <<= 6; - case 0: ch += *source++; - } - ch -= offsetsFromUTF8[extraBytesToRead]; - - if (target >= targetEnd) { - source -= (extraBytesToRead+1); /* Back up source pointer! */ - result = targetExhausted; break; - } - if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */ - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { - if (flags == strictConversion) { - source -= (extraBytesToRead+1); /* return to the illegal value itself */ - result = sourceIllegal; - break; - } else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } else { - *target++ = (UTF16)ch; /* normal case */ - } - } else if (ch > UNI_MAX_UTF16) { - if (flags == strictConversion) { - result = sourceIllegal; - source -= (extraBytesToRead+1); /* return to the start */ - break; /* Bail out; shouldn't continue */ - } else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } else { - /* target is a character in range 0xFFFF - 0x10FFFF. */ - if (target + 1 >= targetEnd) { - source -= (extraBytesToRead+1); /* Back up source pointer! */ - result = targetExhausted; break; - } - ch -= halfBase; - *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START); - *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START); - } - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF32toUTF8 ( - const UTF32** sourceStart, const UTF32* sourceEnd, - UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF32* source = *sourceStart; - UTF8* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch; - unsigned short bytesToWrite = 0; - const UTF32 byteMask = 0xBF; - const UTF32 byteMark = 0x80; - ch = *source++; - if (flags == strictConversion ) { - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } - /* - * Figure out how many bytes the result will require. Turn any - * illegally large UTF32 things (> Plane 17) into replacement chars. - */ - if (ch < (UTF32)0x80) { bytesToWrite = 1; - } else if (ch < (UTF32)0x800) { bytesToWrite = 2; - } else if (ch < (UTF32)0x10000) { bytesToWrite = 3; - } else if (ch <= UNI_MAX_LEGAL_UTF32) { bytesToWrite = 4; - } else { bytesToWrite = 3; - ch = UNI_REPLACEMENT_CHAR; - result = sourceIllegal; - } - - target += bytesToWrite; - if (target > targetEnd) { - --source; /* Back up source pointer! */ - target -= bytesToWrite; result = targetExhausted; break; - } - switch (bytesToWrite) { /* note: everything falls through. */ - case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 1: *--target = (UTF8) (ch | firstByteMark[bytesToWrite]); - } - target += bytesToWrite; - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF8toUTF32 ( - const UTF8** sourceStart, const UTF8* sourceEnd, - UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF8* source = *sourceStart; - UTF32* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch = 0; - unsigned short extraBytesToRead = trailingBytesForUTF8[*source]; - if (source + extraBytesToRead >= sourceEnd) { - result = sourceExhausted; break; - } - /* Do this check whether lenient or strict */ - if (! isLegalUTF8(source, extraBytesToRead+1)) { - result = sourceIllegal; - break; - } - /* - * The cases all fall through. See "Note A" below. - */ - switch (extraBytesToRead) { - case 5: ch += *source++; ch <<= 6; - case 4: ch += *source++; ch <<= 6; - case 3: ch += *source++; ch <<= 6; - case 2: ch += *source++; ch <<= 6; - case 1: ch += *source++; ch <<= 6; - case 0: ch += *source++; - } - ch -= offsetsFromUTF8[extraBytesToRead]; - - if (target >= targetEnd) { - source -= (extraBytesToRead+1); /* Back up the source pointer! */ - result = targetExhausted; break; - } - if (ch <= UNI_MAX_LEGAL_UTF32) { - /* - * UTF-16 surrogate values are illegal in UTF-32, and anything - * over Plane 17 (> 0x10FFFF) is illegal. - */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { - if (flags == strictConversion) { - source -= (extraBytesToRead+1); /* return to the illegal value itself */ - result = sourceIllegal; - break; - } else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } else { - *target++ = ch; - } - } else { /* i.e., ch > UNI_MAX_LEGAL_UTF32 */ - result = sourceIllegal; - *target++ = UNI_REPLACEMENT_CHAR; - } - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- - - Note A. - The fall-through switches in UTF-8 reading code save a - temp variable, some decrements & conditionals. The switches - are equivalent to the following loop: - { - int tmpBytesToRead = extraBytesToRead+1; - do { - ch += *source++; - --tmpBytesToRead; - if (tmpBytesToRead) ch <<= 6; - } while (tmpBytesToRead > 0); - } - In UTF-8 writing code, the switches on "bytesToWrite" are - similarly unrolled loops. - - --------------------------------------------------------------------- */ diff --git a/contrib/ConvertUTF/ConvertUTF.h b/contrib/ConvertUTF/ConvertUTF.h deleted file mode 100644 index 05e800ad8..000000000 --- a/contrib/ConvertUTF/ConvertUTF.h +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright 2001-2004 Unicode, Inc. - * - * Disclaimer - * - * This source code is provided as is by Unicode, Inc. No claims are - * made as to fitness for any particular purpose. No warranties of any - * kind are expressed or implied. The recipient agrees to determine - * applicability of information provided. If this file has been - * purchased on magnetic or optical media from Unicode, Inc., the - * sole remedy for any claim will be exchange of defective media - * within 90 days of receipt. - * - * Limitations on Rights to Redistribute This Code - * - * Unicode, Inc. hereby grants the right to freely use the information - * supplied in this file in the creation of products supporting the - * Unicode Standard, and to make copies of this file in any form - * for internal or external distribution as long as this notice - * remains attached. - */ -#ifndef CONVERTUTF_H -#define CONVERTUTF_H -/* --------------------------------------------------------------------- - - Conversions between UTF32, UTF-16, and UTF-8. Header file. - - Several funtions are included here, forming a complete set of - conversions between the three formats. UTF-7 is not included - here, but is handled in a separate source file. - - Each of these routines takes pointers to input buffers and output - buffers. The input buffers are const. - - Each routine converts the text between *sourceStart and sourceEnd, - putting the result into the buffer between *targetStart and - targetEnd. Note: the end pointers are *after* the last item: e.g. - *(sourceEnd - 1) is the last item. - - The return result indicates whether the conversion was successful, - and if not, whether the problem was in the source or target buffers. - (Only the first encountered problem is indicated.) - - After the conversion, *sourceStart and *targetStart are both - updated to point to the end of last text successfully converted in - the respective buffers. - - Input parameters: - sourceStart - pointer to a pointer to the source buffer. - The contents of this are modified on return so that - it points at the next thing to be converted. - targetStart - similarly, pointer to pointer to the target buffer. - sourceEnd, targetEnd - respectively pointers to the ends of the - two buffers, for overflow checking only. - - These conversion functions take a ConversionFlags argument. When this - flag is set to strict, both irregular sequences and isolated surrogates - will cause an error. When the flag is set to lenient, both irregular - sequences and isolated surrogates are converted. - - Whether the flag is strict or lenient, all illegal sequences will cause - an error return. This includes sequences such as: , , - or in UTF-8, and values above 0x10FFFF in UTF-32. Conformant code - must check for illegal sequences. - - When the flag is set to lenient, characters over 0x10FFFF are converted - to the replacement character; otherwise (when the flag is set to strict) - they constitute an error. - - Output parameters: - The value "sourceIllegal" is returned from some routines if the input - sequence is malformed. When "sourceIllegal" is returned, the source - value will point to the illegal value that caused the problem. E.g., - in UTF-8 when a sequence is malformed, it points to the start of the - malformed sequence. - - Author: Mark E. Davis, 1994. - Rev History: Rick McGowan, fixes & updates May 2001. - Fixes & updates, Sept 2001. - ------------------------------------------------------------------------- */ - -/* --------------------------------------------------------------------- - The following 4 definitions are compiler-specific. - The C standard does not guarantee that wchar_t has at least - 16 bits, so wchar_t is no less portable than unsigned short! - All should be unsigned values to avoid sign extension during - bit mask & shift operations. ------------------------------------------------------------------------- */ - -typedef unsigned long UTF32; /* at least 32 bits */ -typedef unsigned short UTF16; /* at least 16 bits */ -typedef unsigned char UTF8; /* typically 8 bits */ -typedef unsigned char Boolean; /* 0 or 1 */ - -/* Some fundamental constants */ -#define UNI_REPLACEMENT_CHAR (UTF32)0x0000FFFD -#define UNI_MAX_BMP (UTF32)0x0000FFFF -#define UNI_MAX_UTF16 (UTF32)0x0010FFFF -#define UNI_MAX_UTF32 (UTF32)0x7FFFFFFF -#define UNI_MAX_LEGAL_UTF32 (UTF32)0x0010FFFF - -typedef enum { - conversionOK, /* conversion successful */ - sourceExhausted, /* partial character in source, but hit end */ - targetExhausted, /* insuff. room in target for conversion */ - sourceIllegal /* source sequence is illegal/malformed */ -} ConversionResult; - -typedef enum { - strictConversion = 0, - lenientConversion -} ConversionFlags; - -/* This is for C++ and does no harm in C */ -#ifdef __cplusplus -extern "C" { -#endif - -ConversionResult ConvertUTF8toUTF16 ( - const UTF8** sourceStart, const UTF8* sourceEnd, - UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags); - -ConversionResult ConvertUTF16toUTF8 ( - const UTF16** sourceStart, const UTF16* sourceEnd, - UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags); - -ConversionResult ConvertUTF8toUTF32 ( - const UTF8** sourceStart, const UTF8* sourceEnd, - UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags); - -ConversionResult ConvertUTF32toUTF8 ( - const UTF32** sourceStart, const UTF32* sourceEnd, - UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags); - -ConversionResult ConvertUTF16toUTF32 ( - const UTF16** sourceStart, const UTF16* sourceEnd, - UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags); - -ConversionResult ConvertUTF32toUTF16 ( - const UTF32** sourceStart, const UTF32* sourceEnd, - UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags); - -Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd); - -#ifdef __cplusplus -} -#endif - -/* --------------------------------------------------------------------- */ -#endif // CONVERTUTF_H diff --git a/contrib/ConvertUTF/readme.txt b/contrib/ConvertUTF/readme.txt deleted file mode 100644 index b9f17fb81..000000000 --- a/contrib/ConvertUTF/readme.txt +++ /dev/null @@ -1,43 +0,0 @@ - -The accompanying C source code file "ConvertUTF.c" and the associated header -file "ConvertUTF.h" provide for conversion between various transformation -formats of Unicode characters. The following conversions are supported: - - UTF-32 to UTF-16 - UTF-32 to UTF-8 - UTF-16 to UTF-32 - UTF-16 to UTF-8 - UTF-8 to UTF-16 - UTF-8 to UTF-32 - -In addition, there is a test harness which runs various tests. - -The files "CVTUTF7.C" and "CVTUTF7.H" are for archival and historical purposes -only. They have not been updated to Unicode 3.0 or later and should be -considered obsolescent. "CVTUTF7.C" contains two functions that can convert -between UCS2 (i.e., the BMP characters only) and UTF-7. Surrogates are -not supported, the code has not been tested, and should be considered -unsuitable for general purpose use. - -Please submit any bug reports about these programs here: - - http://www.unicode.org/unicode/reporting.html - -Version 1.0: initial version. - -Version 1.1: corrected some minor problems; added stricter checks. - -Version 1.2: corrected switch statements associated with "extraBytesToRead" - in 4 & 5 byte cases, in functions for conversion from UTF8. - Note: formally, the 4 & 5 byte cases are illegal in the latest - UTF8, but the table and this code has always catered for those, - cases since at one time they were legal. - -Version 1.3: Updated UTF-8 legality check; - updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions - Updated UTF-8 legality tests in harness.c - - -Last update: October 19, 2004 - - From 64cbc6933710586f611f571b83e570f350aaf860 Mon Sep 17 00:00:00 2001 From: Kim Kulling Date: Sat, 17 Jun 2017 20:36:59 +0200 Subject: [PATCH 3/3] remove dead code. --- code/BaseImporter.cpp | 40 +------------------------------ code/MMDCpp14.h | 40 +++++++++++++++++++++++++++++++ code/MMDPmdParser.h | 40 +++++++++++++++++++++++++++++++ code/MMDPmxParser.cpp | 50 +++++++++++++++++++++++++++++++-------- code/MMDPmxParser.h | 40 +++++++++++++++++++++++++++++++ code/MMDVmdParser.h | 40 +++++++++++++++++++++++++++++++ code/STEPFileEncoding.cpp | 14 ++--------- 7 files changed, 203 insertions(+), 61 deletions(-) diff --git a/code/BaseImporter.cpp b/code/BaseImporter.cpp index 9a57f860d..7cef43cf6 100644 --- a/code/BaseImporter.cpp +++ b/code/BaseImporter.cpp @@ -305,17 +305,6 @@ void BaseImporter::GetExtensionList(std::set& extensions) #include "../contrib/utf8cpp/source/utf8.h" -// ------------------------------------------------------------------------------------------------ -/*void ReportResult(ConversionResult res) -{ - if(res == sourceExhausted) { - DefaultLogger::get()->error("Source ends with incomplete character sequence, transformation to UTF-8 fails"); - } - else if(res == sourceIllegal) { - DefaultLogger::get()->error("Source contains illegal character sequence, transformation to UTF-8 fails"); - } -}*/ - // ------------------------------------------------------------------------------------------------ // Convert to UTF8 data void BaseImporter::ConvertToUTF8(std::vector& data) @@ -349,23 +338,10 @@ void BaseImporter::ConvertToUTF8(std::vector& data) DefaultLogger::get()->debug("Found UTF-32 BOM ..."); const uint32_t* sstart = (uint32_t*)&data.front()+1, *send = (uint32_t*)&data.back()+1; -// char* dstart,*dend; std::vector output; int *ptr = (int*)&data[ 0 ]; int *end = ptr + ( data.size() / sizeof(int) ) +1; utf8::utf32to8( ptr, end, back_inserter(output)); -/* do { - output.resize(output.size()?output.size()*3/2:data.size()/2); - dstart = &output.front(),dend = &output.back()+1; - - ///result = ConvertUTF32toUTF8((const UTF32**)&sstart,(const UTF32*)send,(UTF8**)&dstart,(UTF8*)dend,lenientConversion); - } while(result == targetExhausted);*/ - - // ReportResult(result); - - // copy to output buffer. -// const size_t outlen = (size_t)(dstart-&output.front()); -// data.assign(output.begin(),output.begin()+outlen); return; } @@ -383,26 +359,12 @@ void BaseImporter::ConvertToUTF8(std::vector& data) DefaultLogger::get()->debug("Found UTF-16 BOM ..."); const uint16_t* sstart = (uint16_t*)&data.front()+1, *send = (uint16_t*)(&data.back()+1); -// char* dstart,*dend; std::vector output; int16_t *ptr = (int16_t*) &data[ 0 ]; int16_t *end = ptr + (data.size() / sizeof(int)) + 1; utf8::utf16to8(data.begin(), data.end(), back_inserter(output)); -/* do { - output.resize(output.size()?output.size()*3/2:data.size()*3/4); - dstart = &output.front(),dend = &output.back()+1; - - utf8::utf16to8(data.begin(), data.end(), output); - - } while(result == targetExhausted); - - ReportResult(result); - - // copy to output buffer. - const size_t outlen = (size_t)(dstart-&output.front()); - data.assign(output.begin(),output.begin()+outlen); - return;*/ + return; } } diff --git a/code/MMDCpp14.h b/code/MMDCpp14.h index ad112de0f..f6f81f827 100644 --- a/code/MMDCpp14.h +++ b/code/MMDCpp14.h @@ -1,3 +1,43 @@ +/* +Open Asset Import Library (assimp) +---------------------------------------------------------------------- + +Copyright (c) 2006-2017, assimp team + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above +copyright notice, this list of conditions and the +following disclaimer. + +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the +following disclaimer in the documentation and/or other +materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its +contributors may be used to endorse or promote products +derived from this software without specific prior +written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +*/ #pragma once #ifndef MMD_CPP14_H diff --git a/code/MMDPmdParser.h b/code/MMDPmdParser.h index 375c323df..44b64585a 100644 --- a/code/MMDPmdParser.h +++ b/code/MMDPmdParser.h @@ -1,3 +1,43 @@ +/* +Open Asset Import Library (assimp) +---------------------------------------------------------------------- + +Copyright (c) 2006-2017, assimp team + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above +copyright notice, this list of conditions and the +following disclaimer. + +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the +following disclaimer in the documentation and/or other +materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its +contributors may be used to endorse or promote products +derived from this software without specific prior +written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +*/ #pragma once #include diff --git a/code/MMDPmxParser.cpp b/code/MMDPmxParser.cpp index 89a4ea6fc..6a76c2c7f 100644 --- a/code/MMDPmxParser.cpp +++ b/code/MMDPmxParser.cpp @@ -1,3 +1,43 @@ +/* +Open Asset Import Library (assimp) +---------------------------------------------------------------------- + +Copyright (c) 2006-2017, assimp team + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above +copyright notice, this list of conditions and the +following disclaimer. + +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the +following disclaimer in the documentation and/or other +materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its +contributors may be used to endorse or promote products +derived from this software without specific prior +written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +*/ #include #include "MMDPmxParser.h" #include "../contrib/utf8cpp/source/utf8.h" @@ -60,16 +100,6 @@ namespace pmx const unsigned int targetSize = size * 3; // enough to encode char* targetStart = new char[targetSize](); const char* targetReserved = targetStart; -// ConversionFlags flags = ConversionFlags::lenientConversion; - - //ConversionResult conversionResult; - /*if( ( conversionResult = ConvertUTF16toUTF8( - (const UTF16**)&sourceStart, (const UTF16*)(sourceStart + size), - (UTF8**)&targetStart, (UTF8*)(targetStart + targetSize), - flags) ) != ConversionResult::conversionOK) { - throw DeadlyImportError( "Convert " + std::string(sourceStart) + " to UTF8 is not valid." ); - }*/ - utf8::utf16to8( sourceStart, sourceStart + size, targetStart ); result.assign(targetReserved, targetStart - targetReserved); diff --git a/code/MMDPmxParser.h b/code/MMDPmxParser.h index 7cb94b9b6..4aff7f8eb 100644 --- a/code/MMDPmxParser.h +++ b/code/MMDPmxParser.h @@ -1,3 +1,43 @@ +/* +Open Asset Import Library (assimp) +---------------------------------------------------------------------- + +Copyright (c) 2006-2017, assimp team + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above +copyright notice, this list of conditions and the +following disclaimer. + +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the +following disclaimer in the documentation and/or other +materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its +contributors may be used to endorse or promote products +derived from this software without specific prior +written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +*/ #pragma once #include diff --git a/code/MMDVmdParser.h b/code/MMDVmdParser.h index 380a10c5d..c5057ef47 100644 --- a/code/MMDVmdParser.h +++ b/code/MMDVmdParser.h @@ -1,3 +1,43 @@ +/* +Open Asset Import Library (assimp) +---------------------------------------------------------------------- + +Copyright (c) 2006-2017, assimp team + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above +copyright notice, this list of conditions and the +following disclaimer. + +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the +following disclaimer in the documentation and/or other +materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its +contributors may be used to endorse or promote products +derived from this software without specific prior +written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +*/ #pragma once #include diff --git a/code/STEPFileEncoding.cpp b/code/STEPFileEncoding.cpp index bb0cb646a..f9a9dd1ce 100644 --- a/code/STEPFileEncoding.cpp +++ b/code/STEPFileEncoding.cpp @@ -40,13 +40,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /** @file STEPFileEncoding.cpp - * @brief STEP character handling, string unescaping + * @brief STEP character handling, string un-escaping */ #include "STEPFileEncoding.h" #include "fast_atof.h" -#include "../contrib/utf8cpp/source/utf8.h" +#include -//#include "../contrib/ConvertUTF/ConvertUTF.h" #include using namespace Assimp; @@ -316,9 +315,6 @@ bool STEP::StringToUTF8(std::string& s) ai_assert(sizeof( unsigned char ) == 1); utf8::utf32to8( univalp, univalp + 1, tempp ); - /*if(ConvertUTF32toUTF8(&univalp, univalp+1, &tempp, tempp+sizeof(temp), lenientConversion) != conversionOK) { - return false; - }*/ const size_t outcount = static_cast(tempp-temp); @@ -371,9 +367,6 @@ bool STEP::StringToUTF8(std::string& s) const uint16_t* srct = src.get(); unsigned char* destt = dest.get(); utf8::utf16to8( srct, srct + count, destt ); - /*if(ConvertUTF16toUTF8(&srct, srct+count, &destt, destt+dcount, lenientConversion) != conversionOK) { - return false; - }*/ const size_t outcount = static_cast(destt-dest.get()); @@ -407,9 +400,6 @@ bool STEP::StringToUTF8(std::string& s) const uint32_t* srct = src.get(); unsigned char* destt = dest.get(); utf8::utf32to8( srct, srct + count, destt ); - /*if(ConvertUTF32toUTF8(&srct, srct+count, &destt, destt+dcount, lenientConversion) != conversionOK) { - return false; - }*/ const size_t outcount = static_cast(destt-dest.get());