Reimplement Unicode.h ; new StringConverter class

9 files changed, 898 insertions, 134 deletions

diff --git a/cpp/src/Ice/Exception.cpp b/cpp/src/Ice/Exception.cpp
index 14a26896a2d..1ea4bcd51dd 100644
--- a/cpp/src/Ice/Exception.cpp
+++ b/cpp/src/Ice/Exception.cpp
@@ -442,6 +442,17 @@ Ice::MemoryLimitException::ice_print(ostream& out) const
 }
 
 void
+Ice::StringConversionException::ice_print(ostream& out) const
+{
+    Exception::ice_print(out);
+    out << ":\nprotocol error: string conversion failed";
+    if(!reason.empty())
+    {
+	out << " (" << reason << ")";
+    }
+}
+
+void
 Ice::EncapsulationException::ice_print(ostream& out) const
 {
     Exception::ice_print(out);
diff --git a/cpp/src/Ice/Makefile b/cpp/src/Ice/Makefile
index b902dfba4b6..d29ac6e0b28 100644
--- a/cpp/src/Ice/Makefile
+++ b/cpp/src/Ice/Makefile
@@ -99,6 +99,7 @@ OBJS		= Acceptor.o \
 		  Stats.o \
 		  StreamI.o \
 		  Stream.o \
+                  StringConvert.o \
 		  SysLoggerI.o \
 		  TcpAcceptor.o \
 		  TcpConnector.o \
diff --git a/cpp/src/Ice/StringConverter.cpp b/cpp/src/Ice/StringConverter.cpp
new file mode 100755
index 00000000000..bf7ed7e4839
--- /dev/null
+++ b/cpp/src/Ice/StringConverter.cpp
@@ -0,0 +1,127 @@
+// **********************************************************************

+//

+// Copyright (c) 2003-2005 ZeroC, Inc. All rights reserved.

+//

+// This copy of Ice is licensed to you under the terms described in the

+// ICE_LICENSE file included in this distribution.

+//

+// **********************************************************************

+

+#include <Ice/StringConverter.h>

+#include <IceUtil/Unicode.h>

+#include <Ice/LocalException.h>

+

+using namespace IceUtil;

+using namespace std;

+

+namespace Ice

+{

+

+Byte* 

+UnicodeWstringConverter::toUTF8(const wchar_t* sourceStart, 

+				const wchar_t* sourceEnd,

+				UTF8Buffer& buffer) const

+{

+    //

+    // The "chunk size" is the maximum of the number of characters in the

+    // source and 6 (== max bytes necessary to encode one Unicode character).

+    //

+    size_t chunkSize = std::max<size_t>(static_cast<size_t>(sourceEnd - sourceStart), 6);

+

+    Byte* targetStart = buffer.getMoreBytes(chunkSize, 0);

+    Byte* targetEnd = targetStart + chunkSize;

+

+    ConversionResult result;

+

+    while((result =

+	  convertUTFWstringToUTF8(sourceStart, sourceEnd, 

+				  targetStart, targetEnd, lenientConversion))

+	  == targetExhausted)

+    {

+	targetStart = buffer.getMoreBytes(chunkSize, targetStart);

+	targetEnd = targetStart + chunkSize;

+    }

+	

+    switch(result)

+    {

+	case conversionOK:

+	    break;

+	case sourceExhausted:

+	    throw StringConversionException(__FILE__, __LINE__, "wide string source exhausted");

+	case sourceIllegal:

+	    throw StringConversionException(__FILE__, __LINE__, "wide string source illegal");

+	default:

+	{

+	    assert(0);

+	    throw StringConversionException(__FILE__, __LINE__);

+	}

+    }

+    return targetStart;

+}

+

+void 

+UnicodeWstringConverter::fromUTF8(const Byte* sourceStart, const Byte* sourceEnd,

+				const wchar_t*& targetStart, const wchar_t*& targetEnd)

+{

+    size_t size = static_cast<size_t>(sourceEnd - sourceStart);

+	

+    wchar_t* outBuf = new wchar_t[size];

+    wchar_t* tStart = outBuf;

+    

+    ConversionResult result = convertUTF8ToUTFWstring(

+	sourceStart, sourceEnd, tStart, outBuf + size, 

+	lenientConversion);

+

+    switch(result)

+    {

+	case conversionOK:

+	{

+	    targetStart = outBuf;

+	    targetEnd = tStart;

+	    break;

+	}

+	case sourceExhausted:

+	    throw StringConversionException(__FILE__, __LINE__, "UTF-8 string source exhausted");

+	case sourceIllegal:

+	    throw StringConversionException(__FILE__, __LINE__, "UTF-8 string source illegal");

+	default:

+	{

+	    assert(0);

+	    throw StringConversionException(__FILE__, __LINE__);

+	}

+    }

+}

+

+void 

+UnicodeWstringConverter::freeTarget(const wchar_t* targetStart) const

+{

+#if defined(_MSC_VER) && _MSC_VER < 1300

+    delete[] const_cast<wchar_t*>(targetStart);

+#else

+    delete[] targetStart;

+#endif

+}

+

+void 

+UnicodeWstringConverter::fromUTF8(const Byte* sourceStart, const Byte* sourceEnd,

+				  wstring& target) const

+{

+    ConversionResult result = 

+	convertUTF8ToUTFWstring(sourceStart, sourceEnd, target, lenientConversion);

+

+    switch(result)

+    {	 

+	case conversionOK:

+	    break;

+	case sourceExhausted:

+	    throw StringConversionException(__FILE__, __LINE__, "UTF-8 string source exhausted");

+	case sourceIllegal:

+	    throw StringConversionException(__FILE__, __LINE__, "UTF-8 string source illegal");

+	default:

+	{

+	    assert(0);

+	    throw StringConversionException(__FILE__, __LINE__);

+	}

+    }

+}

+}

diff --git a/cpp/src/Ice/ice.dsp b/cpp/src/Ice/ice.dsp
index 7c1f15b6d49..53156e29ebf 100644
--- a/cpp/src/Ice/ice.dsp
+++ b/cpp/src/Ice/ice.dsp
@@ -406,6 +406,10 @@ SOURCE=.\StreamI.cpp
 # End Source File

 # Begin Source File

 

+SOURCE=.\StringConverter.cpp

+# End Source File

+# Begin Source File

+

 SOURCE=.\TcpAcceptor.cpp

 # End Source File

 # Begin Source File

@@ -926,6 +930,10 @@ SOURCE=.\StreamI.h
 # End Source File

 # Begin Source File

 

+SOURCE=..\..\include\Ice\StringConverter.h

+# End Source File

+# Begin Source File

+

 SOURCE=.\TcpAcceptor.h

 # End Source File

 # Begin Source File

diff --git a/cpp/src/IceUtil/ConvertUTF.cpp b/cpp/src/IceUtil/ConvertUTF.cpp
new file mode 100755
index 00000000000..91791997432
--- /dev/null
+++ b/cpp/src/IceUtil/ConvertUTF.cpp
@@ -0,0 +1,450 @@
+/*

+ * Copyright 2001-2004 Unicode, Inc.

+ * 

+ * Disclaimer

+ * 

+ * This source code is provided as is by Unicode, Inc. No claims are

+ * made as to fitness for any particular purpose. No warranties of any

+ * kind are expressed or implied. The recipient agrees to determine

+ * applicability of information provided. If this file has been

+ * purchased on magnetic or optical media from Unicode, Inc., the

+ * sole remedy for any claim will be exchange of defective media

+ * within 90 days of receipt.

+ * 

+ * Limitations on Rights to Redistribute This Code

+ * 

+ * Unicode, Inc. hereby grants the right to freely use the information

+ * supplied in this file in the creation of products supporting the

+ * Unicode Standard, and to make copies of this file in any form

+ * for internal or external distribution as long as this notice

+ * remains attached.

+ */

+

+// **********************************************************************

+//

+// Copyright (c) 2006 ZeroC, Inc. All rights reserved.

+//

+// This file was slightly modified to move the definitions to the

+// IceUtil namespace and use C++ types.

+//

+// **********************************************************************

+

+

+

+/* ---------------------------------------------------------------------

+

+    Conversions between UTF32, UTF-16, and UTF-8. Source code file.

+    Author: Mark E. Davis, 1994.

+    Rev History: Rick McGowan, fixes & updates May 2001.

+    Sept 2001: fixed const & error conditions per

+	mods suggested by S. Parent & A. Lillich.

+    June 2002: Tim Dodd added detection and handling of incomplete

+	source sequences, enhanced error detection, added casts

+	to eliminate compiler warnings.

+    July 2003: slight mods to back out aggressive FFFE detection.

+    Jan 2004: updated switches in from-UTF8 conversions.

+    Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions.

+

+    See the header file "ConvertUTF.h" for complete documentation.

+

+------------------------------------------------------------------------ */

+

+

+#include <IceUtil/ConvertUTF.h>

+

+#ifdef CVTUTF_DEBUG

+#include <stdio.h>

+#endif

+

+namespace IceUtil

+{

+

+static const int halfShift  = 10; /* used for shifting by 10 bits */

+

+static const UTF32 halfBase = 0x0010000UL;

+static const UTF32 halfMask = 0x3FFUL;

+

+#define UNI_SUR_HIGH_START  (UTF32)0xD800

+#define UNI_SUR_HIGH_END    (UTF32)0xDBFF

+#define UNI_SUR_LOW_START   (UTF32)0xDC00

+#define UNI_SUR_LOW_END     (UTF32)0xDFFF

+// #define false	   0

+// #define true	    1

+

+

+/* --------------------------------------------------------------------- */

+

+/*

+ * Index into the table below with the first byte of a UTF-8 sequence to

+ * get the number of trailing bytes that are supposed to follow it.

+ * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is

+ * left as-is for anyone who may want to do such conversion, which was

+ * allowed in earlier algorithms.

+ */

+static const char trailingBytesForUTF8[256] = {

+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,

+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,

+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,

+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,

+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,

+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,

+    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,

+    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5

+};

+

+/*

+ * Magic values subtracted from a buffer value during UTF8 conversion.

+ * This table contains as many values as there might be trailing bytes

+ * in a UTF-8 sequence.

+ */

+static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, 

+		     0x03C82080UL, 0xFA082080UL, 0x82082080UL };

+

+/*

+ * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed

+ * into the first byte, depending on how many bytes follow.  There are

+ * as many entries in this table as there are UTF-8 sequence types.

+ * (I.e., one byte sequence, two byte... etc.). Remember that sequencs

+ * for *legal* UTF-8 will be 4 or fewer bytes total.

+ */

+static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };

+

+/* --------------------------------------------------------------------- */

+

+/* The interface converts a whole buffer to avoid function-call overhead.

+ * Constants have been gathered. Loops & conditionals have been removed as

+ * much as possible for efficiency, in favor of drop-through switches.

+ * (See "Note A" at the bottom of the file for equivalent code.)

+ * If your compiler supports it, the "isLegalUTF8" call can be turned

+ * into an inline function.

+ */

+

+/* --------------------------------------------------------------------- */

+

+ConversionResult ConvertUTF16toUTF8 (

+	const UTF16** sourceStart, const UTF16* sourceEnd, 

+	UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {

+    ConversionResult result = conversionOK;

+    const UTF16* source = *sourceStart;

+    UTF8* target = *targetStart;

+    while (source < sourceEnd) {

+	UTF32 ch;

+	unsigned short bytesToWrite = 0;

+	const UTF32 byteMask = 0xBF;

+	const UTF32 byteMark = 0x80; 

+	const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */

+	ch = *source++;

+	/* If we have a surrogate pair, convert to UTF32 first. */

+	if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {

+	    /* If the 16 bits following the high surrogate are in the source buffer... */

+	    if (source < sourceEnd) {

+		UTF32 ch2 = *source;

+		/* If it's a low surrogate, convert to UTF32. */

+		if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {

+		    ch = ((ch - UNI_SUR_HIGH_START) << halfShift)

+			+ (ch2 - UNI_SUR_LOW_START) + halfBase;

+		    ++source;

+		} else if (flags == strictConversion) { /* it's an unpaired high surrogate */

+		    --source; /* return to the illegal value itself */

+		    result = sourceIllegal;

+		    break;

+		}

+	    } else { /* We don't have the 16 bits following the high surrogate. */

+		--source; /* return to the high surrogate */

+		result = sourceExhausted;

+		break;

+	    }

+	} else if (flags == strictConversion) {

+	    /* UTF-16 surrogate values are illegal in UTF-32 */

+	    if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {

+		--source; /* return to the illegal value itself */

+		result = sourceIllegal;

+		break;

+	    }

+	}

+	/* Figure out how many bytes the result will require */

+	if (ch < (UTF32)0x80) {	     bytesToWrite = 1;

+	} else if (ch < (UTF32)0x800) {     bytesToWrite = 2;

+	} else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;

+	} else if (ch < (UTF32)0x110000) {  bytesToWrite = 4;

+	} else {			    bytesToWrite = 3;

+					    ch = UNI_REPLACEMENT_CHAR;

+	}

+

+	target += bytesToWrite;

+	if (target > targetEnd) {

+	    source = oldSource; /* Back up source pointer! */

+	    target -= bytesToWrite; result = targetExhausted; break;

+	}

+	switch (bytesToWrite) { /* note: everything falls through. */

+	    case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;

+	    case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;

+	    case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;

+	    case 1: *--target =  (UTF8)(ch | firstByteMark[bytesToWrite]);

+	}

+	target += bytesToWrite;

+    }

+    *sourceStart = source;

+    *targetStart = target;

+    return result;

+}

+

+/* --------------------------------------------------------------------- */

+

+/*

+ * Utility routine to tell whether a sequence of bytes is legal UTF-8.

+ * This must be called with the length pre-determined by the first byte.

+ * If not calling this from ConvertUTF8to*, then the length can be set by:

+ *  length = trailingBytesForUTF8[*source]+1;

+ * and the sequence is illegal right away if there aren't that many bytes

+ * available.

+ * If presented with a length > 4, this returns false.  The Unicode

+ * definition of UTF-8 goes up to 4-byte sequences.

+ */

+

+static Boolean isLegalUTF8(const UTF8 *source, int length) {

+    UTF8 a;

+    const UTF8 *srcptr = source+length;

+    switch (length) {

+    default: return false;

+	/* Everything else falls through when "true"... */

+    case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;

+    case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;

+    case 2: if ((a = (*--srcptr)) > 0xBF) return false;

+

+	switch (*source) {

+	    /* no fall-through in this inner switch */

+	    case 0xE0: if (a < 0xA0) return false; break;

+	    case 0xED: if (a > 0x9F) return false; break;

+	    case 0xF0: if (a < 0x90) return false; break;

+	    case 0xF4: if (a > 0x8F) return false; break;

+	    default:   if (a < 0x80) return false;

+	}

+

+    case 1: if (*source >= 0x80 && *source < 0xC2) return false;

+    }

+    if (*source > 0xF4) return false;

+    return true;

+}

+

+/* --------------------------------------------------------------------- */

+

+/*

+ * Exported function to return whether a UTF-8 sequence is legal or not.

+ * This is not used here; it's just exported.

+ */

+Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) {

+    int length = trailingBytesForUTF8[*source]+1;

+    if (source+length > sourceEnd) {

+	return false;

+    }

+    return isLegalUTF8(source, length);

+}

+

+/* --------------------------------------------------------------------- */

+

+ConversionResult ConvertUTF8toUTF16 (

+	const UTF8** sourceStart, const UTF8* sourceEnd, 

+	UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) {

+    ConversionResult result = conversionOK;

+    const UTF8* source = *sourceStart;

+    UTF16* target = *targetStart;

+    while (source < sourceEnd) {

+	UTF32 ch = 0;

+	unsigned short extraBytesToRead = trailingBytesForUTF8[*source];

+	if (source + extraBytesToRead >= sourceEnd) {

+	    result = sourceExhausted; break;

+	}

+	/* Do this check whether lenient or strict */

+	if (! isLegalUTF8(source, extraBytesToRead+1)) {

+	    result = sourceIllegal;

+	    break;

+	}

+	/*

+	 * The cases all fall through. See "Note A" below.

+	 */

+	switch (extraBytesToRead) {

+	    case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */

+	    case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */

+	    case 3: ch += *source++; ch <<= 6;

+	    case 2: ch += *source++; ch <<= 6;

+	    case 1: ch += *source++; ch <<= 6;

+	    case 0: ch += *source++;

+	}

+	ch -= offsetsFromUTF8[extraBytesToRead];

+

+	if (target >= targetEnd) {

+	    source -= (extraBytesToRead+1); /* Back up source pointer! */

+	    result = targetExhausted; break;

+	}

+	if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */

+	    /* UTF-16 surrogate values are illegal in UTF-32 */

+	    if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {

+		if (flags == strictConversion) {

+		    source -= (extraBytesToRead+1); /* return to the illegal value itself */

+		    result = sourceIllegal;

+		    break;

+		} else {

+		    *target++ = UNI_REPLACEMENT_CHAR;

+		}

+	    } else {

+		*target++ = (UTF16)ch; /* normal case */

+	    }

+	} else if (ch > UNI_MAX_UTF16) {

+	    if (flags == strictConversion) {

+		result = sourceIllegal;

+		source -= (extraBytesToRead+1); /* return to the start */

+		break; /* Bail out; shouldn't continue */

+	    } else {

+		*target++ = UNI_REPLACEMENT_CHAR;

+	    }

+	} else {

+	    /* target is a character in range 0xFFFF - 0x10FFFF. */

+	    if (target + 1 >= targetEnd) {

+		source -= (extraBytesToRead+1); /* Back up source pointer! */

+		result = targetExhausted; break;

+	    }

+	    ch -= halfBase;

+	    *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);

+	    *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);

+	}

+    }

+    *sourceStart = source;

+    *targetStart = target;

+    return result;

+}

+

+/* --------------------------------------------------------------------- */

+

+ConversionResult ConvertUTF32toUTF8 (

+	const UTF32** sourceStart, const UTF32* sourceEnd, 

+	UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {

+    ConversionResult result = conversionOK;

+    const UTF32* source = *sourceStart;

+    UTF8* target = *targetStart;

+    while (source < sourceEnd) {

+	UTF32 ch;

+	unsigned short bytesToWrite = 0;

+	const UTF32 byteMask = 0xBF;

+	const UTF32 byteMark = 0x80; 

+	ch = *source++;

+	if (flags == strictConversion ) {

+	    /* UTF-16 surrogate values are illegal in UTF-32 */

+	    if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {

+		--source; /* return to the illegal value itself */

+		result = sourceIllegal;

+		break;

+	    }

+	}

+	/*

+	 * Figure out how many bytes the result will require. Turn any

+	 * illegally large UTF32 things (> Plane 17) into replacement chars.

+	 */

+	if (ch < (UTF32)0x80) {	     bytesToWrite = 1;

+	} else if (ch < (UTF32)0x800) {     bytesToWrite = 2;

+	} else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;

+	} else if (ch <= UNI_MAX_LEGAL_UTF32) {  bytesToWrite = 4;

+	} else {			    bytesToWrite = 3;

+					    ch = UNI_REPLACEMENT_CHAR;

+					    result = sourceIllegal;

+	}

+	

+	target += bytesToWrite;

+	if (target > targetEnd) {

+	    --source; /* Back up source pointer! */

+	    target -= bytesToWrite; result = targetExhausted; break;

+	}

+	switch (bytesToWrite) { /* note: everything falls through. */

+	    case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;

+	    case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;

+	    case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;

+	    case 1: *--target = (UTF8) (ch | firstByteMark[bytesToWrite]);

+	}

+	target += bytesToWrite;

+    }

+    *sourceStart = source;

+    *targetStart = target;

+    return result;

+}

+

+/* --------------------------------------------------------------------- */

+

+ConversionResult ConvertUTF8toUTF32 (

+	const UTF8** sourceStart, const UTF8* sourceEnd, 

+	UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) {

+    ConversionResult result = conversionOK;

+    const UTF8* source = *sourceStart;

+    UTF32* target = *targetStart;

+    while (source < sourceEnd) {

+	UTF32 ch = 0;

+	unsigned short extraBytesToRead = trailingBytesForUTF8[*source];

+	if (source + extraBytesToRead >= sourceEnd) {

+	    result = sourceExhausted; break;

+	}

+	/* Do this check whether lenient or strict */

+	if (! isLegalUTF8(source, extraBytesToRead+1)) {

+	    result = sourceIllegal;

+	    break;

+	}

+	/*

+	 * The cases all fall through. See "Note A" below.

+	 */

+	switch (extraBytesToRead) {

+	    case 5: ch += *source++; ch <<= 6;

+	    case 4: ch += *source++; ch <<= 6;

+	    case 3: ch += *source++; ch <<= 6;

+	    case 2: ch += *source++; ch <<= 6;

+	    case 1: ch += *source++; ch <<= 6;

+	    case 0: ch += *source++;

+	}

+	ch -= offsetsFromUTF8[extraBytesToRead];

+

+	if (target >= targetEnd) {

+	    source -= (extraBytesToRead+1); /* Back up the source pointer! */

+	    result = targetExhausted; break;

+	}

+	if (ch <= UNI_MAX_LEGAL_UTF32) {

+	    /*

+	     * UTF-16 surrogate values are illegal in UTF-32, and anything

+	     * over Plane 17 (> 0x10FFFF) is illegal.

+	     */

+	    if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {

+		if (flags == strictConversion) {

+		    source -= (extraBytesToRead+1); /* return to the illegal value itself */

+		    result = sourceIllegal;

+		    break;

+		} else {

+		    *target++ = UNI_REPLACEMENT_CHAR;

+		}

+	    } else {

+		*target++ = ch;

+	    }

+	} else { /* i.e., ch > UNI_MAX_LEGAL_UTF32 */

+	    result = sourceIllegal;

+	    *target++ = UNI_REPLACEMENT_CHAR;

+	}

+    }

+    *sourceStart = source;

+    *targetStart = target;

+    return result;

+}

+

+/* ---------------------------------------------------------------------

+

+    Note A.

+    The fall-through switches in UTF-8 reading code save a

+    temp variable, some decrements & conditionals.  The switches

+    are equivalent to the following loop:

+	{

+	    int tmpBytesToRead = extraBytesToRead+1;

+	    do {

+		ch += *source++;

+		--tmpBytesToRead;

+		if (tmpBytesToRead) ch <<= 6;

+	    } while (tmpBytesToRead > 0);

+	}

+    In UTF-8 writing code, the switches on "bytesToWrite" are

+    similarly unrolled loops.

+

+   --------------------------------------------------------------------- */

+}

diff --git a/cpp/src/IceUtil/ConvertUTF.h b/cpp/src/IceUtil/ConvertUTF.h
new file mode 100755
index 00000000000..64922cf67d1
--- /dev/null
+++ b/cpp/src/IceUtil/ConvertUTF.h
@@ -0,0 +1,144 @@
+/*

+ * Copyright 2001-2004 Unicode, Inc.

+ * 

+ * Disclaimer

+ * 

+ * This source code is provided as is by Unicode, Inc. No claims are

+ * made as to fitness for any particular purpose. No warranties of any

+ * kind are expressed or implied. The recipient agrees to determine

+ * applicability of information provided. If this file has been

+ * purchased on magnetic or optical media from Unicode, Inc., the

+ * sole remedy for any claim will be exchange of defective media

+ * within 90 days of receipt.

+ * 

+ * Limitations on Rights to Redistribute This Code

+ * 

+ * Unicode, Inc. hereby grants the right to freely use the information

+ * supplied in this file in the creation of products supporting the

+ * Unicode Standard, and to make copies of this file in any form

+ * for internal or external distribution as long as this notice

+ * remains attached.

+ */

+

+// **********************************************************************

+//

+// Copyright (c) 2006 ZeroC, Inc. All rights reserved.

+//

+// This file was slightly modified to move the definitions to the

+// IceUtil namespace and use C++ types.

+//

+// **********************************************************************

+

+#ifndef ICE_UTIL_CONVERT_UTF_H

+#define ICE_UTIL_CONVERT_UTF_H

+

+#include <IceUtil/Unicode.h>

+

+

+/* ---------------------------------------------------------------------

+

+    Conversions between UTF32, UTF-16, and UTF-8.  Header file.

+

+    Several funtions are included here, forming a complete set of

+    conversions between the three formats.  UTF-7 is not included

+    here, but is handled in a separate source file.

+

+    Each of these routines takes pointers to input buffers and output

+    buffers.  The input buffers are const.

+

+    Each routine converts the text between *sourceStart and sourceEnd,

+    putting the result into the buffer between *targetStart and

+    targetEnd. Note: the end pointers are *after* the last item: e.g. 

+    *(sourceEnd - 1) is the last item.

+

+    The return result indicates whether the conversion was successful,

+    and if not, whether the problem was in the source or target buffers.

+    (Only the first encountered problem is indicated.)

+

+    After the conversion, *sourceStart and *targetStart are both

+    updated to point to the end of last text successfully converted in

+    the respective buffers.

+

+    Input parameters:

+	sourceStart - pointer to a pointer to the source buffer.

+		The contents of this are modified on return so that

+		it points at the next thing to be converted.

+	targetStart - similarly, pointer to pointer to the target buffer.

+	sourceEnd, targetEnd - respectively pointers to the ends of the

+		two buffers, for overflow checking only.

+

+    These conversion functions take a ConversionFlags argument. When this

+    flag is set to strict, both irregular sequences and isolated surrogates

+    will cause an error.  When the flag is set to lenient, both irregular

+    sequences and isolated surrogates are converted.

+

+    Whether the flag is strict or lenient, all illegal sequences will cause

+    an error return. This includes sequences such as: <F4 90 80 80>, <C0 80>,

+    or <A0> in UTF-8, and values above 0x10FFFF in UTF-32. Conformant code

+    must check for illegal sequences.

+

+    When the flag is set to lenient, characters over 0x10FFFF are converted

+    to the replacement character; otherwise (when the flag is set to strict)

+    they constitute an error.

+

+    Output parameters:

+	The value "sourceIllegal" is returned from some routines if the input

+	sequence is malformed.  When "sourceIllegal" is returned, the source

+	value will point to the illegal value that caused the problem. E.g.,

+	in UTF-8 when a sequence is malformed, it points to the start of the

+	malformed sequence.  

+

+    Author: Mark E. Davis, 1994.

+    Rev History: Rick McGowan, fixes & updates May 2001.

+		 Fixes & updates, Sept 2001.

+

+------------------------------------------------------------------------ */

+

+/* ---------------------------------------------------------------------

+    The following 4 definitions are compiler-specific.

+    The C standard does not guarantee that wchar_t has at least

+    16 bits, so wchar_t is no less portable than unsigned short!

+    All should be unsigned values to avoid sign extension during

+    bit mask & shift operations.

+------------------------------------------------------------------------ */

+

+namespace IceUtil

+{

+

+typedef unsigned int	UTF32;	/* at least 32 bits */

+typedef unsigned short	UTF16;	/* at least 16 bits */

+typedef unsigned char	UTF8;	/* typically 8 bits */

+typedef bool	        Boolean; /* 0 or 1 */

+

+/* Some fundamental constants */

+#define UNI_REPLACEMENT_CHAR (UTF32)0x0000FFFD

+#define UNI_MAX_BMP (UTF32)0x0000FFFF

+#define UNI_MAX_UTF16 (UTF32)0x0010FFFF

+#define UNI_MAX_UTF32 (UTF32)0x7FFFFFFF

+#define UNI_MAX_LEGAL_UTF32 (UTF32)0x0010FFFF

+

+

+ConversionResult ConvertUTF8toUTF16(

+    const UTF8** sourceStart, const UTF8* sourceEnd, 

+    UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags);

+

+ConversionResult ConvertUTF16toUTF8 (

+    const UTF16** sourceStart, const UTF16* sourceEnd, 

+    UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags);

+		

+ConversionResult ConvertUTF8toUTF32(

+    const UTF8** sourceStart, const UTF8* sourceEnd, 

+    UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags);

+

+ConversionResult ConvertUTF32toUTF8(

+    const UTF32** sourceStart, const UTF32* sourceEnd, 

+    UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags);

+

+//

+// isLegalUTFSequence is declared in IceUtil/Unicode.h

+//

+

+/* --------------------------------------------------------------------- */

+

+}

+#endif

diff --git a/cpp/src/IceUtil/Makefile b/cpp/src/IceUtil/Makefile
index a1b4ce376f3..9992e4c2c0c 100644
--- a/cpp/src/IceUtil/Makefile
+++ b/cpp/src/IceUtil/Makefile
@@ -17,6 +17,7 @@ TARGETS		= $(call mklibtargets,$(libdir)/$(LIBFILENAME),$(libdir)/$(SONAME),$(li
 
 OBJS		= Base64.o \
 		  Cond.o \
+                  ConvertUTF.o \
 		  CountDownLatch.o \
 		  CtrlCHandler.o \
 		  Exception.o \
diff --git a/cpp/src/IceUtil/Unicode.cpp b/cpp/src/IceUtil/Unicode.cpp
index 724139f25bd..287ddbe1e7a 100644
--- a/cpp/src/IceUtil/Unicode.cpp
+++ b/cpp/src/IceUtil/Unicode.cpp
@@ -8,151 +8,169 @@
 // **********************************************************************
 
 #include <IceUtil/Unicode.h>
-#include <algorithm>
-
-#if defined(_WIN32)
-#   define SIZEOF_WCHAR_T 2
-#elif (defined(__sun) && defined(__sparc)) || \
-      ((defined(__linux) || defined(__FreeBSD__)) && (defined(__i386) || defined(__x86_64)))
-#   define SIZEOF_WCHAR_T 4
-#endif
+#include <IceUtil/ConvertUTF.h>
 
 using namespace std;
+using namespace IceUtil;
 
-string
-IceUtil::wstringToString(const wstring& str)
+namespace
+{
+//
+// Helper class, base never defined
+// Usage: WstringHelper<sizeof(wchar_t)>::toUTF8 and fromUTF8.
+//
+template<size_t wcharSize> 
+struct WstringHelper
 {
-    string result;
-    result.reserve(str.length() * 2);
+    static ConversionResult toUTF8(
+	const wchar_t*& sourceStart, const wchar_t* sourceEnd, 
+	Byte*& targetStart, Byte* targetEnd, ConversionFlags flags);
 
-    for(unsigned int i = 0; i < str.length(); ++i)
+    static ConversionResult fromUTF8(
+	const Byte*& sourceStart, const Byte* sourceEnd, 
+	wchar_t*& targetStart, wchar_t* targetEnd, ConversionFlags flags);
+};
+
+template<>
+struct WstringHelper<2>
+{
+    static ConversionResult toUTF8(
+	const wchar_t*& sourceStart, const wchar_t* sourceEnd, 
+	Byte*& targetStart, Byte* targetEnd, ConversionFlags flags)
     {
-	wchar_t wc;
-	wc = str[i];
-
-	if(wc < 0x80)
-	{
-	    result += static_cast<char>(wc);
-	}
-	else if(wc < 0x800)
-	{
-	    result += 0xc0 | (wc>>6);
-	    result += 0x80 | (wc & 0x3f);
-	}
-	else if(wc < 0x10000)
-	{
-	    result += 0xe0 | (wc>>12);
-	    result += 0x80 | ((wc>>6) & 0x3f);
-	    result += 0x80 | (wc & 0x3f);
-	}
-#if SIZEOF_WCHAR_T >= 4
-	else if(wc < 0x10FFFF)
-	{
-	    result += 0xf0 | (wc>>18);
-	    result += 0x80 | ((wc>>12) & 0x3f);
-	    result += 0x80 | ((wc>>6) & 0x3f);
-	    result += 0x80 | (wc & 0x3f);
-	}
-#endif
-	else
-	{
-	    return result; // Error, not encodable.
-	}
+	return ConvertUTF16toUTF8(
+	    reinterpret_cast<const UTF16**>(&sourceStart),
+	    reinterpret_cast<const UTF16*>(sourceEnd),
+	    &targetStart, targetEnd, flags);
     }
+    
+    static ConversionResult fromUTF8(
+	const Byte*& sourceStart, const Byte* sourceEnd, 
+	wchar_t*& targetStart, wchar_t* targetEnd, ConversionFlags flags)
+    {
+	return ConvertUTF8toUTF16(
+	    &sourceStart, sourceEnd,
+	    reinterpret_cast<UTF16**>(&targetStart),
+	    reinterpret_cast<UTF16*>(targetEnd), flags);
+    }
+};
 
-    return result;
+template<>
+struct WstringHelper<4>
+{
+    static ConversionResult toUTF8(
+	const wchar_t*& sourceStart, const wchar_t* sourceEnd, 
+	Byte*& targetStart, Byte* targetEnd, ConversionFlags flags)
+    {
+	return ConvertUTF32toUTF8(
+	    reinterpret_cast<const UTF32**>(&sourceStart),
+	    reinterpret_cast<const UTF32*>(sourceEnd),
+	    &targetStart, targetEnd, flags);
+    }
+    
+    static ConversionResult fromUTF8(
+	const Byte*& sourceStart, const Byte* sourceEnd, 
+	wchar_t*& targetStart, wchar_t* targetEnd, ConversionFlags flags)
+    {
+	return ConvertUTF8toUTF32(
+	    &sourceStart, sourceEnd,
+	    reinterpret_cast<UTF32**>(&targetStart),
+	    reinterpret_cast<UTF32*>(targetEnd), flags);
+    }
+};
 }
 
-wstring
-IceUtil::stringToWstring(const string& str)
+//
+// convertXXX functions
+//
+
+ConversionResult 
+IceUtil::convertUTFWstringToUTF8(
+    const wchar_t*& sourceStart, const wchar_t* sourceEnd, 
+    Byte*& targetStart, Byte* targetEnd, ConversionFlags flags)
 {
-    wstring result;
-    result.reserve(str.length());
+    return WstringHelper<sizeof(wchar_t)>::toUTF8(
+	sourceStart, sourceEnd, targetStart, targetEnd, flags);
+}
+
+ConversionResult
+IceUtil::convertUTF8ToUTFWstring(
+    const Byte*& sourceStart, const Byte* sourceEnd, 
+    wchar_t*& targetStart, wchar_t* targetEnd, ConversionFlags flags)
+{
+    return WstringHelper<sizeof(wchar_t)>::fromUTF8(
+	sourceStart, sourceEnd, targetStart, targetEnd, flags);
+}
 
-    unsigned int len;
-    for(unsigned int i = 0; i < str.length(); i += len)
+ConversionResult 
+IceUtil::convertUTF8ToUTFWstring(const Byte*& sourceStart, const Byte* sourceEnd, 
+				 std::wstring& target, ConversionFlags flags)
+{
+    //
+    // Could be reimplemented without this temporary wchar_t buffer
+    //
+    size_t size = static_cast<size_t>(sourceEnd - sourceStart);
+    wchar_t* outBuf = new wchar_t[size];
+    wchar_t* targetStart = outBuf; 
+    wchar_t* targetEnd = targetStart + size;
+
+    ConversionResult result =  
+	convertUTF8ToUTFWstring(sourceStart, sourceEnd, targetStart,
+				targetEnd, flags);
+
+    if(result == conversionOK)
     {
-	unsigned char c = str[i];
-	wchar_t wc;
-	int minval;
-
-	if(c < 0x80)
-	{
-	    wc = c;
-	    len = 1;
-	    minval = 0;
-	}
-	else if(c < 0xc0) // Lead byte must not be 10xxxxxx
-	{
-	    return result; // Error, not encodable.
-	}
-	else if(c < 0xe0) // 110xxxxx
-	{
-	    wc = c & 0x1f;
-	    len = 2;
-	    minval = 0x80;
-	}
-	else if(c < 0xf0) // 1110xxxx
-	{
-	    wc = c & 0xf;
-	    len = 3;
-	    minval = 0x800;
-	}
-#if SIZEOF_WCHAR_T >= 4
-	else if(c < 0xf8) // 11110xxx
-	{
-	    wc = c & 7;
-	    len = 4;
-	    minval = 0x10000;
-	}
-	else if(c < 0xfc) // 111110xx
-	{
-	    // Length 5 and 6 is declared invalid in Unicode 3.1 and ISO 10646:2003.
-	    wc = c & 3;
-	    len = 5;
-	    minval = 0x110000;
-	}
-	else if(c < 0xfe) // 1111110x
-	{
-	    // Length 5 and 6 is declared invalid in Unicode 3.1 and ISO 10646:2003.
-	    wc = c & 1;
-	    len = 6;
-	    minval = 0x4000000;
-	}
-#endif
-	else
-	{
-	    return result; // Error, not encodable.
-	}
-
-	if(i + len - 1 < str.length())
-	{
-	    for(unsigned int j = 1; j < len; ++j)
-	    {
-		if((str[i + j] & 0xc0) != 0x80) // All other bytes must be 10xxxxxx
-		{
-		    return result; // Error, not encodable.
-		}
-		
-		wc <<= 6;
-		wc |= str[i + j] & 0x3f;
-	    }
-
-	    if(wc < minval)
-	    {
-		return result; // Error, non-shortest form.
-	    }
-	    else
-	    {
-		result += wc;
-	    }
-	}
-	else
-	{
-	    return result; // Error, not encodable.
-	}
+	std::wstring s(outBuf, static_cast<size_t>(targetStart - outBuf));
+	s.swap(target);
+    }
+    delete[] outBuf;
+    return result;
+}
+
+
+//
+// wstringToString and stringToWstring
+//
+
+string
+IceUtil::wstringToString(const wstring& wstr)
+{
+    string target;
+    
+    size_t size = wstr.size() * 3 * (sizeof(wchar_t) / 2);
+
+    Byte* outBuf = new Byte[size];
+    Byte* targetStart = outBuf; 
+    Byte* targetEnd = outBuf + size;
+
+    const wchar_t* sourceStart = wstr.data();
+  
+    ConversionResult result = 
+	convertUTFWstringToUTF8(
+	    sourceStart, sourceStart + wstr.size(), 
+	    targetStart, targetEnd, lenientConversion);
+	
+    if(result == conversionOK)
+    {
+	string s(reinterpret_cast<char*>(outBuf),
+		 static_cast<size_t>(targetStart - outBuf));
+	s.swap(target);
     }
+    delete[] outBuf;
+    return target;
+}
 
+wstring
+IceUtil::stringToWstring(const string& str)
+{
+    wstring result;
+    const Byte* sourceStart = reinterpret_cast<const Byte*>(str.data());
+    
+    convertUTF8ToUTFWstring(sourceStart, sourceStart + str.size(),
+			    result, lenientConversion);
+    //
+    // TODO: check the ConversionResult and do something with it!
+    //
     return result;
 }
 
@@ -166,28 +184,24 @@ IceUtil::stringToWstring(const string& str)
 string
 IceUtil::wstringToString(const basic_string<__wchar_t>& str)
 {
-    assert(sizeof(__wchar_t) == SIZEOF_WCHAR_T);
     return wstringToString(*reinterpret_cast<const wstring*>(&str));
 }
 
 basic_string<__wchar_t>
 IceUtil::stringToNativeWstring(const string& str)
 {
-    assert(sizeof(__wchar_t) == SIZEOF_WCHAR_T);
     return reinterpret_cast<basic_string<__wchar_t>& >(stringToWstring(str));
 }
 #   else
 string
 IceUtil::wstringToString(const basic_string<unsigned short>& str)
 {
-    assert(sizeof(__wchar_t) == SIZEOF_WCHAR_T);
     return wstringToString(*reinterpret_cast<const wstring*>(&str));
 }
 
 basic_string<unsigned short>
 IceUtil::stringToTypedefWstring(const string& str)
 {
-    assert(sizeof(__wchar_t) == SIZEOF_WCHAR_T);
     return reinterpret_cast<basic_string<unsigned short>& >(stringToWstring(str));
 }
 
diff --git a/cpp/src/IceUtil/iceutil.dsp b/cpp/src/IceUtil/iceutil.dsp
index 19a8f9c9978..21946bb1288 100644
--- a/cpp/src/IceUtil/iceutil.dsp
+++ b/cpp/src/IceUtil/iceutil.dsp
@@ -112,6 +112,10 @@ SOURCE=.\Cond.cpp
 # End Source File

 # Begin Source File

 

+SOURCE=.\ConvertUTF.cpp

+# End Source File

+# Begin Source File

+

 SOURCE=.\CountDownLatch.cpp

 # End Source File

 # Begin Source File

@@ -200,6 +204,10 @@ SOURCE=..\..\include\iceutil\Config.h
 # End Source File

 # Begin Source File

 

+SOURCE=..\..\include\IceUtil\ConvertUTF.h

+# End Source File

+# Begin Source File

+

 SOURCE=..\..\include\iceutil\CountDownLatch.h

 # End Source File

 # Begin Source File