Utf8: integrate new lib.

code/BaseImporter.cpp

@@ -303,7 +303,6 @@ void BaseImporter::GetExtensionList(std::set<std::string>& extensions)
     return false;
 }
 
-//#include "../contrib/ConvertUTF/ConvertUTF.h"
 #include "../contrib/utf8cpp/source/utf8.h"
 
 // ------------------------------------------------------------------------------------------------

code/BaseImporter.h

@@ -61,7 +61,6 @@ class BaseProcess;
 class SharedPostProcessInfo;
 class IOStream;
 
-
 // utility to do char4 to uint32 in a portable manner
 #define AI_MAKE_MAGIC(string) ((uint32_t)((string[0] << 24) + \
     (string[1] << 16) + (string[2] << 8) + string[3]))
@@ -194,14 +193,11 @@ public:
         const Importer* pImp
         );
 
-
     // -------------------------------------------------------------------
     /** Called by #Importer::GetImporterInfo to get a description of
      *  some loader features. Importers must provide this information. */
     virtual const aiImporterDesc* GetInfo() const = 0;
 
-
-
     // -------------------------------------------------------------------
     /** Called by #Importer::GetExtensionList for each loaded importer.
      *  Take the extension list contained in the structure returned by
@@ -317,7 +313,7 @@ public: // static utilities
      *  @param Size of one token, in bytes. Maximally 16 bytes.
      *  @return true if one of the given tokens was found
      *
-     *  @note For convinence, the check is also performed for the
+     *  @note For convenience, the check is also performed for the
      *  byte-swapped variant of all tokens (big endian). Only for
      *  tokens of size 2,4.
      */

code/CMakeLists.txt

@@ -695,11 +695,6 @@ SET( Extra_SRCS
 )
 SOURCE_GROUP( Extra FILES ${Extra_SRCS})
 
-SET( ConvertUTF_SRCS
-  ../contrib/ConvertUTF/ConvertUTF.h
-  ../contrib/ConvertUTF/ConvertUTF.c
-)
-SOURCE_GROUP( ConvertUTF FILES ${ConvertUTF_SRCS})
 
 SET( Clipper_SRCS
   ../contrib/clipper/clipper.hpp
@@ -835,7 +830,6 @@ SET( assimp_src
 
   # Third-party libraries
   ${IrrXML_SRCS}
-  ${ConvertUTF_SRCS}
   ${unzip_compile_SRCS}
   ${Poly2Tri_SRCS}
   ${Clipper_SRCS}

code/MMDPmxParser.cpp

@@ -1,7 +1,7 @@
 #include <utility>
 #include "MMDPmxParser.h"
+#include "../contrib/utf8cpp/source/utf8.h"
 #include "Exceptional.h"
-#include "../contrib/ConvertUTF/ConvertUTF.h"
 
 namespace pmx
 {
@@ -60,15 +60,17 @@ namespace pmx
 			const unsigned int targetSize = size * 3; // enough to encode
 			char* targetStart = new char[targetSize]();
 			const char* targetReserved = targetStart;
-			ConversionFlags flags = ConversionFlags::lenientConversion;
+//			ConversionFlags flags = ConversionFlags::lenientConversion;
 
-			ConversionResult conversionResult;
+			//ConversionResult conversionResult;
-			if( ( conversionResult = ConvertUTF16toUTF8(
+			/*if( ( conversionResult = ConvertUTF16toUTF8(
 				(const UTF16**)&sourceStart, (const UTF16*)(sourceStart + size),
 				(UTF8**)&targetStart, (UTF8*)(targetStart + targetSize),
 				flags) ) != ConversionResult::conversionOK) {
 				throw DeadlyImportError( "Convert " + std::string(sourceStart) + " to UTF8 is not valid." );
-			}
+			}*/
+
+            utf8::utf16to8( sourceStart, sourceStart + size, targetStart );
 
 			result.assign(targetReserved, targetStart - targetReserved);
 			delete[] targetReserved;

code/STEPFileEncoding.cpp

@@ -44,14 +44,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "STEPFileEncoding.h"
 #include "fast_atof.h"
+#include "../contrib/utf8cpp/source/utf8.h"
 
-#include "../contrib/ConvertUTF/ConvertUTF.h"
+//#include "../contrib/ConvertUTF/ConvertUTF.h"
 #include <memory>
 
 using namespace Assimp;
 
 // roman1 to utf16 table
-static const UTF16 mac_codetable[] = {
+static const uint16_t mac_codetable[] = {
     // 0x20 unassig./nonprint. slots
      0x0020 ,
      0x0021 ,
@@ -309,14 +310,15 @@ bool STEP::StringToUTF8(std::string& s)
 
                 ai_assert(sizeof(mac_codetable) / sizeof(mac_codetable[0]) == 0x100-0x20);
 
-                const UTF32 unival = mac_codetable[macval - 0x20], *univalp = &unival;
+                const uint32_t unival = mac_codetable[macval - 0x20], *univalp = &unival;
 
-                UTF8 temp[5], *tempp = temp;
+                unsigned char temp[5], *tempp = temp;
-                ai_assert(sizeof(UTF8) == 1);
+                ai_assert(sizeof( unsigned char ) == 1);
 
-                if(ConvertUTF32toUTF8(&univalp, univalp+1, &tempp, tempp+sizeof(temp), lenientConversion) != conversionOK) {
+                utf8::utf32to8( univalp, univalp + 1, tempp );
+                /*if(ConvertUTF32toUTF8(&univalp, univalp+1, &tempp, tempp+sizeof(temp), lenientConversion) != conversionOK) {
                     return false;
-                }
+                }*/
 
                 const size_t outcount = static_cast<size_t>(tempp-temp);
 
@@ -355,28 +357,29 @@ bool STEP::StringToUTF8(std::string& s)
                             }
 
                             const size_t count = (j-basei)/4;
-                            std::unique_ptr<UTF16[]> src(new UTF16[count]);
+                            std::unique_ptr<uint16_t[]> src(new uint16_t[count]);
 
                             const char* cur = s.c_str() + basei;
                             for (size_t k = 0; k < count; ++k, cur += 4) {
-                                src[k] = (static_cast<UTF16>(HexOctetToDecimal(cur)) << 8u)  |
+                                src[k] = (static_cast<uint16_t>(HexOctetToDecimal(cur)) << 8u)  |
-                                     static_cast<UTF16>(HexOctetToDecimal(cur+2));
+                                     static_cast<uint16_t>(HexOctetToDecimal(cur+2));
                             }
 
                             const size_t dcount = count * 3; // this is enough to hold all possible outputs
-                            std::unique_ptr<UTF8[]> dest(new UTF8[dcount]);
+                            std::unique_ptr<unsigned char[]> dest(new unsigned char[dcount]);
 
-                            const UTF16* srct = src.get();
+                            const uint16_t* srct = src.get();
-                            UTF8* destt = dest.get();
+                            unsigned char* destt = dest.get();
-                            if(ConvertUTF16toUTF8(&srct, srct+count, &destt, destt+dcount, lenientConversion) != conversionOK) {
+                            utf8::utf16to8( srct, srct + count, destt );
+                            /*if(ConvertUTF16toUTF8(&srct, srct+count, &destt, destt+dcount, lenientConversion) != conversionOK) {
                                 return false;
-                            }
+                            }*/
 
                             const size_t outcount = static_cast<size_t>(destt-dest.get());
 
                             s.erase(i,(j+4-i));
 
-                            ai_assert(sizeof(UTF8) == 1);
+                            ai_assert(sizeof(unsigned char) == 1);
                             s.insert(i, reinterpret_cast<char*>(dest.get()), outcount);
 
                             i += outcount;
@@ -388,37 +391,37 @@ bool STEP::StringToUTF8(std::string& s)
                             }
 
                             const size_t count = (j-basei)/8;
-                            std::unique_ptr<UTF32[]> src(new UTF32[count]);
+                            std::unique_ptr<uint32_t[]> src(new uint32_t[count]);
 
                             const char* cur = s.c_str() + basei;
                             for (size_t k = 0; k < count; ++k, cur += 8) {
-                                src[k] = (static_cast<UTF32>(HexOctetToDecimal(cur  )) << 24u) |
+                                src[k] = (static_cast<uint32_t>(HexOctetToDecimal(cur  )) << 24u) |
-                                         (static_cast<UTF32>(HexOctetToDecimal(cur+2)) << 16u) |
+                                         (static_cast<uint32_t>(HexOctetToDecimal(cur+2)) << 16u) |
-                                         (static_cast<UTF32>(HexOctetToDecimal(cur+4)) << 8u)  |
+                                         (static_cast<uint32_t>(HexOctetToDecimal(cur+4)) << 8u)  |
-                                         (static_cast<UTF32>(HexOctetToDecimal(cur+6)));
+                                         (static_cast<uint32_t>(HexOctetToDecimal(cur+6)));
                             }
 
                             const size_t dcount = count * 5; // this is enough to hold all possible outputs
-                            std::unique_ptr<UTF8[]> dest(new UTF8[dcount]);
+                            std::unique_ptr<unsigned char[]> dest(new unsigned char[dcount]);
 
-                            const UTF32* srct = src.get();
+                            const uint32_t* srct = src.get();
-                            UTF8* destt = dest.get();
+                            unsigned char* destt = dest.get();
-                            if(ConvertUTF32toUTF8(&srct, srct+count, &destt, destt+dcount, lenientConversion) != conversionOK) {
+                            utf8::utf32to8( srct, srct + count, destt );
+                            /*if(ConvertUTF32toUTF8(&srct, srct+count, &destt, destt+dcount, lenientConversion) != conversionOK) {
                                 return false;
-                            }
+                            }*/
 
                             const size_t outcount = static_cast<size_t>(destt-dest.get());
 
                             s.erase(i,(j+4-i));
 
-                            ai_assert(sizeof(UTF8) == 1);
+                            ai_assert(sizeof(unsigned char) == 1);
                             s.insert(i, reinterpret_cast<char*>(dest.get()), outcount);
 
                             i += outcount;
                             continue;
                         }
                     }
-
                     break;
 
                     // TODO: other encoding patterns?

contrib/ConvertUTF/ConvertUTF.c

@@ -1,539 +0,0 @@
-/*
- * Copyright 2001-2004 Unicode, Inc.
- * 
- * Disclaimer
- * 
- * This source code is provided as is by Unicode, Inc. No claims are
- * made as to fitness for any particular purpose. No warranties of any
- * kind are expressed or implied. The recipient agrees to determine
- * applicability of information provided. If this file has been
- * purchased on magnetic or optical media from Unicode, Inc., the
- * sole remedy for any claim will be exchange of defective media
- * within 90 days of receipt.
- * 
- * Limitations on Rights to Redistribute This Code
- * 
- * Unicode, Inc. hereby grants the right to freely use the information
- * supplied in this file in the creation of products supporting the
- * Unicode Standard, and to make copies of this file in any form
- * for internal or external distribution as long as this notice
- * remains attached.
- */
-
-/* ---------------------------------------------------------------------
-
-    Conversions between UTF32, UTF-16, and UTF-8. Source code file.
-    Author: Mark E. Davis, 1994.
-    Rev History: Rick McGowan, fixes & updates May 2001.
-    Sept 2001: fixed const & error conditions per
-	mods suggested by S. Parent & A. Lillich.
-    June 2002: Tim Dodd added detection and handling of incomplete
-	source sequences, enhanced error detection, added casts
-	to eliminate compiler warnings.
-    July 2003: slight mods to back out aggressive FFFE detection.
-    Jan 2004: updated switches in from-UTF8 conversions.
-    Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions.
-
-    See the header file "ConvertUTF.h" for complete documentation.
-
------------------------------------------------------------------------- */
-
-
-#include "ConvertUTF.h"
-#ifdef CVTUTF_DEBUG
-#include <stdio.h>
-#endif
-
-static const int halfShift  = 10; /* used for shifting by 10 bits */
-
-static const UTF32 halfBase = 0x0010000UL;
-static const UTF32 halfMask = 0x3FFUL;
-
-#define UNI_SUR_HIGH_START  (UTF32)0xD800
-#define UNI_SUR_HIGH_END    (UTF32)0xDBFF
-#define UNI_SUR_LOW_START   (UTF32)0xDC00
-#define UNI_SUR_LOW_END     (UTF32)0xDFFF
-#define false	   0
-#define true	    1
-
-/* --------------------------------------------------------------------- */
-
-ConversionResult ConvertUTF32toUTF16 (
-	const UTF32** sourceStart, const UTF32* sourceEnd, 
-	UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) {
-    ConversionResult result = conversionOK;
-    const UTF32* source = *sourceStart;
-    UTF16* target = *targetStart;
-    while (source < sourceEnd) {
-	UTF32 ch;
-	if (target >= targetEnd) {
-	    result = targetExhausted; break;
-	}
-	ch = *source++;
-	if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
-	    /* UTF-16 surrogate values are illegal in UTF-32; 0xffff or 0xfffe are both reserved values */
-	    if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
-		if (flags == strictConversion) {
-		    --source; /* return to the illegal value itself */
-		    result = sourceIllegal;
-		    break;
-		} else {
-		    *target++ = UNI_REPLACEMENT_CHAR;
-		}
-	    } else {
-		*target++ = (UTF16)ch; /* normal case */
-	    }
-	} else if (ch > UNI_MAX_LEGAL_UTF32) {
-	    if (flags == strictConversion) {
-		result = sourceIllegal;
-	    } else {
-		*target++ = UNI_REPLACEMENT_CHAR;
-	    }
-	} else {
-	    /* target is a character in range 0xFFFF - 0x10FFFF. */
-	    if (target + 1 >= targetEnd) {
-		--source; /* Back up source pointer! */
-		result = targetExhausted; break;
-	    }
-	    ch -= halfBase;
-	    *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
-	    *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
-	}
-    }
-    *sourceStart = source;
-    *targetStart = target;
-    return result;
-}
-
-/* --------------------------------------------------------------------- */
-
-ConversionResult ConvertUTF16toUTF32 (
-	const UTF16** sourceStart, const UTF16* sourceEnd, 
-	UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) {
-    ConversionResult result = conversionOK;
-    const UTF16* source = *sourceStart;
-    UTF32* target = *targetStart;
-    UTF32 ch, ch2;
-    while (source < sourceEnd) {
-	const UTF16* oldSource = source; /*  In case we have to back up because of target overflow. */
-	ch = *source++;
-	/* If we have a surrogate pair, convert to UTF32 first. */
-	if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
-	    /* If the 16 bits following the high surrogate are in the source buffer... */
-	    if (source < sourceEnd) {
-		ch2 = *source;
-		/* If it's a low surrogate, convert to UTF32. */
-		if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
-		    ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
-			+ (ch2 - UNI_SUR_LOW_START) + halfBase;
-		    ++source;
-		} else if (flags == strictConversion) { /* it's an unpaired high surrogate */
-		    --source; /* return to the illegal value itself */
-		    result = sourceIllegal;
-		    break;
-		}
-	    } else { /* We don't have the 16 bits following the high surrogate. */
-		--source; /* return to the high surrogate */
-		result = sourceExhausted;
-		break;
-	    }
-	} else if (flags == strictConversion) {
-	    /* UTF-16 surrogate values are illegal in UTF-32 */
-	    if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
-		--source; /* return to the illegal value itself */
-		result = sourceIllegal;
-		break;
-	    }
-	}
-	if (target >= targetEnd) {
-	    source = oldSource; /* Back up source pointer! */
-	    result = targetExhausted; break;
-	}
-	*target++ = ch;
-    }
-    *sourceStart = source;
-    *targetStart = target;
-#ifdef CVTUTF_DEBUG
-if (result == sourceIllegal) {
-    fprintf(stderr, "ConvertUTF16toUTF32 illegal seq 0x%04x,%04x\n", ch, ch2);
-    fflush(stderr);
-}
-#endif
-    return result;
-}
-
-/* --------------------------------------------------------------------- */
-
-/*
- * Index into the table below with the first byte of a UTF-8 sequence to
- * get the number of trailing bytes that are supposed to follow it.
- * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is
- * left as-is for anyone who may want to do such conversion, which was
- * allowed in earlier algorithms.
- */
-static const char trailingBytesForUTF8[256] = {
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
-};
-
-/*
- * Magic values subtracted from a buffer value during UTF8 conversion.
- * This table contains as many values as there might be trailing bytes
- * in a UTF-8 sequence.
- */
-static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, 
-		     0x03C82080UL, 0xFA082080UL, 0x82082080UL };
-
-/*
- * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed
- * into the first byte, depending on how many bytes follow.  There are
- * as many entries in this table as there are UTF-8 sequence types.
- * (I.e., one byte sequence, two byte... etc.). Remember that sequencs
- * for *legal* UTF-8 will be 4 or fewer bytes total.
- */
-static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
-
-/* --------------------------------------------------------------------- */
-
-/* The interface converts a whole buffer to avoid function-call overhead.
- * Constants have been gathered. Loops & conditionals have been removed as
- * much as possible for efficiency, in favor of drop-through switches.
- * (See "Note A" at the bottom of the file for equivalent code.)
- * If your compiler supports it, the "isLegalUTF8" call can be turned
- * into an inline function.
- */
-
-/* --------------------------------------------------------------------- */
-
-ConversionResult ConvertUTF16toUTF8 (
-	const UTF16** sourceStart, const UTF16* sourceEnd, 
-	UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
-    ConversionResult result = conversionOK;
-    const UTF16* source = *sourceStart;
-    UTF8* target = *targetStart;
-    while (source < sourceEnd) {
-	UTF32 ch;
-	unsigned short bytesToWrite = 0;
-	const UTF32 byteMask = 0xBF;
-	const UTF32 byteMark = 0x80; 
-	const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */
-	ch = *source++;
-	/* If we have a surrogate pair, convert to UTF32 first. */
-	if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
-	    /* If the 16 bits following the high surrogate are in the source buffer... */
-	    if (source < sourceEnd) {
-		UTF32 ch2 = *source;
-		/* If it's a low surrogate, convert to UTF32. */
-		if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
-		    ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
-			+ (ch2 - UNI_SUR_LOW_START) + halfBase;
-		    ++source;
-		} else if (flags == strictConversion) { /* it's an unpaired high surrogate */
-		    --source; /* return to the illegal value itself */
-		    result = sourceIllegal;
-		    break;
-		}
-	    } else { /* We don't have the 16 bits following the high surrogate. */
-		--source; /* return to the high surrogate */
-		result = sourceExhausted;
-		break;
-	    }
-	} else if (flags == strictConversion) {
-	    /* UTF-16 surrogate values are illegal in UTF-32 */
-	    if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
-		--source; /* return to the illegal value itself */
-		result = sourceIllegal;
-		break;
-	    }
-	}
-	/* Figure out how many bytes the result will require */
-	if (ch < (UTF32)0x80) {	     bytesToWrite = 1;
-	} else if (ch < (UTF32)0x800) {     bytesToWrite = 2;
-	} else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;
-	} else if (ch < (UTF32)0x110000) {  bytesToWrite = 4;
-	} else {			    bytesToWrite = 3;
-					    ch = UNI_REPLACEMENT_CHAR;
-	}
-
-	target += bytesToWrite;
-	if (target > targetEnd) {
-	    source = oldSource; /* Back up source pointer! */
-	    target -= bytesToWrite; result = targetExhausted; break;
-	}
-	switch (bytesToWrite) { /* note: everything falls through. */
-	    case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
-	    case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
-	    case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
-	    case 1: *--target =  (UTF8)(ch | firstByteMark[bytesToWrite]);
-	}
-	target += bytesToWrite;
-    }
-    *sourceStart = source;
-    *targetStart = target;
-    return result;
-}
-
-/* --------------------------------------------------------------------- */
-
-/*
- * Utility routine to tell whether a sequence of bytes is legal UTF-8.
- * This must be called with the length pre-determined by the first byte.
- * If not calling this from ConvertUTF8to*, then the length can be set by:
- *  length = trailingBytesForUTF8[*source]+1;
- * and the sequence is illegal right away if there aren't that many bytes
- * available.
- * If presented with a length > 4, this returns false.  The Unicode
- * definition of UTF-8 goes up to 4-byte sequences.
- */
-
-static Boolean isLegalUTF8(const UTF8 *source, int length) {
-    UTF8 a;
-    const UTF8 *srcptr = source+length;
-    switch (length) {
-    default: return false;
-	/* Everything else falls through when "true"... */
-    case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
-    case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
-    case 2: if ((a = (*--srcptr)) > 0xBF) return false;
-
-	switch (*source) {
-	    /* no fall-through in this inner switch */
-	    case 0xE0: if (a < 0xA0) return false; break;
-	    case 0xED: if (a > 0x9F) return false; break;
-	    case 0xF0: if (a < 0x90) return false; break;
-	    case 0xF4: if (a > 0x8F) return false; break;
-	    default:   if (a < 0x80) return false;
-	}
-
-    case 1: if (*source >= 0x80 && *source < 0xC2) return false;
-    }
-    if (*source > 0xF4) return false;
-    return true;
-}
-
-/* --------------------------------------------------------------------- */
-
-/*
- * Exported function to return whether a UTF-8 sequence is legal or not.
- * This is not used here; it's just exported.
- */
-Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) {
-    int length = trailingBytesForUTF8[*source]+1;
-    if (source+length > sourceEnd) {
-	return false;
-    }
-    return isLegalUTF8(source, length);
-}
-
-/* --------------------------------------------------------------------- */
-
-ConversionResult ConvertUTF8toUTF16 (
-	const UTF8** sourceStart, const UTF8* sourceEnd, 
-	UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) {
-    ConversionResult result = conversionOK;
-    const UTF8* source = *sourceStart;
-    UTF16* target = *targetStart;
-    while (source < sourceEnd) {
-	UTF32 ch = 0;
-	unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
-	if (source + extraBytesToRead >= sourceEnd) {
-	    result = sourceExhausted; break;
-	}
-	/* Do this check whether lenient or strict */
-	if (! isLegalUTF8(source, extraBytesToRead+1)) {
-	    result = sourceIllegal;
-	    break;
-	}
-	/*
-	 * The cases all fall through. See "Note A" below.
-	 */
-	switch (extraBytesToRead) {
-	    case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
-	    case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
-	    case 3: ch += *source++; ch <<= 6;
-	    case 2: ch += *source++; ch <<= 6;
-	    case 1: ch += *source++; ch <<= 6;
-	    case 0: ch += *source++;
-	}
-	ch -= offsetsFromUTF8[extraBytesToRead];
-
-	if (target >= targetEnd) {
-	    source -= (extraBytesToRead+1); /* Back up source pointer! */
-	    result = targetExhausted; break;
-	}
-	if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
-	    /* UTF-16 surrogate values are illegal in UTF-32 */
-	    if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
-		if (flags == strictConversion) {
-		    source -= (extraBytesToRead+1); /* return to the illegal value itself */
-		    result = sourceIllegal;
-		    break;
-		} else {
-		    *target++ = UNI_REPLACEMENT_CHAR;
-		}
-	    } else {
-		*target++ = (UTF16)ch; /* normal case */
-	    }
-	} else if (ch > UNI_MAX_UTF16) {
-	    if (flags == strictConversion) {
-		result = sourceIllegal;
-		source -= (extraBytesToRead+1); /* return to the start */
-		break; /* Bail out; shouldn't continue */
-	    } else {
-		*target++ = UNI_REPLACEMENT_CHAR;
-	    }
-	} else {
-	    /* target is a character in range 0xFFFF - 0x10FFFF. */
-	    if (target + 1 >= targetEnd) {
-		source -= (extraBytesToRead+1); /* Back up source pointer! */
-		result = targetExhausted; break;
-	    }
-	    ch -= halfBase;
-	    *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
-	    *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
-	}
-    }
-    *sourceStart = source;
-    *targetStart = target;
-    return result;
-}
-
-/* --------------------------------------------------------------------- */
-
-ConversionResult ConvertUTF32toUTF8 (
-	const UTF32** sourceStart, const UTF32* sourceEnd, 
-	UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
-    ConversionResult result = conversionOK;
-    const UTF32* source = *sourceStart;
-    UTF8* target = *targetStart;
-    while (source < sourceEnd) {
-	UTF32 ch;
-	unsigned short bytesToWrite = 0;
-	const UTF32 byteMask = 0xBF;
-	const UTF32 byteMark = 0x80; 
-	ch = *source++;
-	if (flags == strictConversion ) {
-	    /* UTF-16 surrogate values are illegal in UTF-32 */
-	    if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
-		--source; /* return to the illegal value itself */
-		result = sourceIllegal;
-		break;
-	    }
-	}
-	/*
-	 * Figure out how many bytes the result will require. Turn any
-	 * illegally large UTF32 things (> Plane 17) into replacement chars.
-	 */
-	if (ch < (UTF32)0x80) {	     bytesToWrite = 1;
-	} else if (ch < (UTF32)0x800) {     bytesToWrite = 2;
-	} else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;
-	} else if (ch <= UNI_MAX_LEGAL_UTF32) {  bytesToWrite = 4;
-	} else {			    bytesToWrite = 3;
-					    ch = UNI_REPLACEMENT_CHAR;
-					    result = sourceIllegal;
-	}
-	
-	target += bytesToWrite;
-	if (target > targetEnd) {
-	    --source; /* Back up source pointer! */
-	    target -= bytesToWrite; result = targetExhausted; break;
-	}
-	switch (bytesToWrite) { /* note: everything falls through. */
-	    case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
-	    case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
-	    case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
-	    case 1: *--target = (UTF8) (ch | firstByteMark[bytesToWrite]);
-	}
-	target += bytesToWrite;
-    }
-    *sourceStart = source;
-    *targetStart = target;
-    return result;
-}
-
-/* --------------------------------------------------------------------- */
-
-ConversionResult ConvertUTF8toUTF32 (
-	const UTF8** sourceStart, const UTF8* sourceEnd, 
-	UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) {
-    ConversionResult result = conversionOK;
-    const UTF8* source = *sourceStart;
-    UTF32* target = *targetStart;
-    while (source < sourceEnd) {
-	UTF32 ch = 0;
-	unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
-	if (source + extraBytesToRead >= sourceEnd) {
-	    result = sourceExhausted; break;
-	}
-	/* Do this check whether lenient or strict */
-	if (! isLegalUTF8(source, extraBytesToRead+1)) {
-	    result = sourceIllegal;
-	    break;
-	}
-	/*
-	 * The cases all fall through. See "Note A" below.
-	 */
-	switch (extraBytesToRead) {
-	    case 5: ch += *source++; ch <<= 6;
-	    case 4: ch += *source++; ch <<= 6;
-	    case 3: ch += *source++; ch <<= 6;
-	    case 2: ch += *source++; ch <<= 6;
-	    case 1: ch += *source++; ch <<= 6;
-	    case 0: ch += *source++;
-	}
-	ch -= offsetsFromUTF8[extraBytesToRead];
-
-	if (target >= targetEnd) {
-	    source -= (extraBytesToRead+1); /* Back up the source pointer! */
-	    result = targetExhausted; break;
-	}
-	if (ch <= UNI_MAX_LEGAL_UTF32) {
-	    /*
-	     * UTF-16 surrogate values are illegal in UTF-32, and anything
-	     * over Plane 17 (> 0x10FFFF) is illegal.
-	     */
-	    if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
-		if (flags == strictConversion) {
-		    source -= (extraBytesToRead+1); /* return to the illegal value itself */
-		    result = sourceIllegal;
-		    break;
-		} else {
-		    *target++ = UNI_REPLACEMENT_CHAR;
-		}
-	    } else {
-		*target++ = ch;
-	    }
-	} else { /* i.e., ch > UNI_MAX_LEGAL_UTF32 */
-	    result = sourceIllegal;
-	    *target++ = UNI_REPLACEMENT_CHAR;
-	}
-    }
-    *sourceStart = source;
-    *targetStart = target;
-    return result;
-}
-
-/* ---------------------------------------------------------------------
-
-    Note A.
-    The fall-through switches in UTF-8 reading code save a
-    temp variable, some decrements & conditionals.  The switches
-    are equivalent to the following loop:
-	{
-	    int tmpBytesToRead = extraBytesToRead+1;
-	    do {
-		ch += *source++;
-		--tmpBytesToRead;
-		if (tmpBytesToRead) ch <<= 6;
-	    } while (tmpBytesToRead > 0);
-	}
-    In UTF-8 writing code, the switches on "bytesToWrite" are
-    similarly unrolled loops.
-
-   --------------------------------------------------------------------- */

contrib/ConvertUTF/ConvertUTF.h

@@ -1,151 +0,0 @@
-/*
- * Copyright 2001-2004 Unicode, Inc.
- * 
- * Disclaimer
- * 
- * This source code is provided as is by Unicode, Inc. No claims are
- * made as to fitness for any particular purpose. No warranties of any
- * kind are expressed or implied. The recipient agrees to determine
- * applicability of information provided. If this file has been
- * purchased on magnetic or optical media from Unicode, Inc., the
- * sole remedy for any claim will be exchange of defective media
- * within 90 days of receipt.
- * 
- * Limitations on Rights to Redistribute This Code
- * 
- * Unicode, Inc. hereby grants the right to freely use the information
- * supplied in this file in the creation of products supporting the
- * Unicode Standard, and to make copies of this file in any form
- * for internal or external distribution as long as this notice
- * remains attached.
- */
-#ifndef CONVERTUTF_H
-#define CONVERTUTF_H
-/* ---------------------------------------------------------------------
-
-    Conversions between UTF32, UTF-16, and UTF-8.  Header file.
-
-    Several funtions are included here, forming a complete set of
-    conversions between the three formats.  UTF-7 is not included
-    here, but is handled in a separate source file.
-
-    Each of these routines takes pointers to input buffers and output
-    buffers.  The input buffers are const.
-
-    Each routine converts the text between *sourceStart and sourceEnd,
-    putting the result into the buffer between *targetStart and
-    targetEnd. Note: the end pointers are *after* the last item: e.g. 
-    *(sourceEnd - 1) is the last item.
-
-    The return result indicates whether the conversion was successful,
-    and if not, whether the problem was in the source or target buffers.
-    (Only the first encountered problem is indicated.)
-
-    After the conversion, *sourceStart and *targetStart are both
-    updated to point to the end of last text successfully converted in
-    the respective buffers.
-
-    Input parameters:
-	sourceStart - pointer to a pointer to the source buffer.
-		The contents of this are modified on return so that
-		it points at the next thing to be converted.
-	targetStart - similarly, pointer to pointer to the target buffer.
-	sourceEnd, targetEnd - respectively pointers to the ends of the
-		two buffers, for overflow checking only.
-
-    These conversion functions take a ConversionFlags argument. When this
-    flag is set to strict, both irregular sequences and isolated surrogates
-    will cause an error.  When the flag is set to lenient, both irregular
-    sequences and isolated surrogates are converted.
-
-    Whether the flag is strict or lenient, all illegal sequences will cause
-    an error return. This includes sequences such as: <F4 90 80 80>, <C0 80>,
-    or <A0> in UTF-8, and values above 0x10FFFF in UTF-32. Conformant code
-    must check for illegal sequences.
-
-    When the flag is set to lenient, characters over 0x10FFFF are converted
-    to the replacement character; otherwise (when the flag is set to strict)
-    they constitute an error.
-
-    Output parameters:
-	The value "sourceIllegal" is returned from some routines if the input
-	sequence is malformed.  When "sourceIllegal" is returned, the source
-	value will point to the illegal value that caused the problem. E.g.,
-	in UTF-8 when a sequence is malformed, it points to the start of the
-	malformed sequence.  
-
-    Author: Mark E. Davis, 1994.
-    Rev History: Rick McGowan, fixes & updates May 2001.
-		 Fixes & updates, Sept 2001.
-
------------------------------------------------------------------------- */
-
-/* ---------------------------------------------------------------------
-    The following 4 definitions are compiler-specific.
-    The C standard does not guarantee that wchar_t has at least
-    16 bits, so wchar_t is no less portable than unsigned short!
-    All should be unsigned values to avoid sign extension during
-    bit mask & shift operations.
------------------------------------------------------------------------- */
-
-typedef unsigned long	UTF32;	/* at least 32 bits */
-typedef unsigned short	UTF16;	/* at least 16 bits */
-typedef unsigned char	UTF8;	/* typically 8 bits */
-typedef unsigned char	Boolean; /* 0 or 1 */
-
-/* Some fundamental constants */
-#define UNI_REPLACEMENT_CHAR (UTF32)0x0000FFFD
-#define UNI_MAX_BMP (UTF32)0x0000FFFF
-#define UNI_MAX_UTF16 (UTF32)0x0010FFFF
-#define UNI_MAX_UTF32 (UTF32)0x7FFFFFFF
-#define UNI_MAX_LEGAL_UTF32 (UTF32)0x0010FFFF
-
-typedef enum {
-	conversionOK, 		/* conversion successful */
-	sourceExhausted,	/* partial character in source, but hit end */
-	targetExhausted,	/* insuff. room in target for conversion */
-	sourceIllegal		/* source sequence is illegal/malformed */
-} ConversionResult;
-
-typedef enum {
-	strictConversion = 0,
-	lenientConversion
-} ConversionFlags;
-
-/* This is for C++ and does no harm in C */
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-ConversionResult ConvertUTF8toUTF16 (
-		const UTF8** sourceStart, const UTF8* sourceEnd, 
-		UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags);
-
-ConversionResult ConvertUTF16toUTF8 (
-		const UTF16** sourceStart, const UTF16* sourceEnd, 
-		UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags);
-		
-ConversionResult ConvertUTF8toUTF32 (
-		const UTF8** sourceStart, const UTF8* sourceEnd, 
-		UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags);
-
-ConversionResult ConvertUTF32toUTF8 (
-		const UTF32** sourceStart, const UTF32* sourceEnd, 
-		UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags);
-		
-ConversionResult ConvertUTF16toUTF32 (
-		const UTF16** sourceStart, const UTF16* sourceEnd, 
-		UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags);
-
-ConversionResult ConvertUTF32toUTF16 (
-		const UTF32** sourceStart, const UTF32* sourceEnd, 
-		UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags);
-
-Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd);
-
-#ifdef __cplusplus
-}
-#endif
-
-/* --------------------------------------------------------------------- */
-#endif  // CONVERTUTF_H

contrib/ConvertUTF/readme.txt

@@ -1,43 +0,0 @@
-
-The accompanying C source code file "ConvertUTF.c" and the associated header
-file "ConvertUTF.h" provide for conversion between various transformation
-formats of Unicode characters.  The following conversions are supported:
-
-	UTF-32 to UTF-16
-	UTF-32 to UTF-8
-	UTF-16 to UTF-32
-	UTF-16 to UTF-8
-	UTF-8 to UTF-16
-	UTF-8 to UTF-32
-
-In addition, there is a test harness which runs various tests.
-
-The files "CVTUTF7.C" and "CVTUTF7.H" are for archival and historical purposes
-only. They have not been updated to Unicode 3.0 or later and should be
-considered obsolescent. "CVTUTF7.C" contains two functions that can convert
-between UCS2 (i.e., the BMP characters only) and UTF-7. Surrogates are
-not supported, the code has not been tested, and should be considered
-unsuitable for general purpose use.
-
-Please submit any bug reports about these programs here:
-
-	http://www.unicode.org/unicode/reporting.html
-
-Version 1.0: initial version.
-
-Version 1.1: corrected some minor problems; added stricter checks.
-
-Version 1.2: corrected switch statements associated with "extraBytesToRead"
-	in 4 & 5 byte cases, in functions for conversion from UTF8.
-	Note: formally, the 4 & 5 byte cases are illegal in the latest
-	UTF8, but the table and this code has always catered for those,
-	cases since at one time they were legal.
-
-Version 1.3: Updated UTF-8 legality check;
-	updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions
-	Updated UTF-8 legality tests in harness.c
- 
-
-Last update: October 19, 2004
-
-