diff options
Diffstat (limited to 'cpp/src/IceUtil/Unicode.cpp')
| -rw-r--r-- | cpp/src/IceUtil/Unicode.cpp | 282 |
1 files changed, 148 insertions, 134 deletions
diff --git a/cpp/src/IceUtil/Unicode.cpp b/cpp/src/IceUtil/Unicode.cpp index 724139f25bd..287ddbe1e7a 100644 --- a/cpp/src/IceUtil/Unicode.cpp +++ b/cpp/src/IceUtil/Unicode.cpp @@ -8,151 +8,169 @@ // ********************************************************************** #include <IceUtil/Unicode.h> -#include <algorithm> - -#if defined(_WIN32) -# define SIZEOF_WCHAR_T 2 -#elif (defined(__sun) && defined(__sparc)) || \ - ((defined(__linux) || defined(__FreeBSD__)) && (defined(__i386) || defined(__x86_64))) -# define SIZEOF_WCHAR_T 4 -#endif +#include <IceUtil/ConvertUTF.h> using namespace std; +using namespace IceUtil; -string -IceUtil::wstringToString(const wstring& str) +namespace +{ +// +// Helper class, base never defined +// Usage: WstringHelper<sizeof(wchar_t)>::toUTF8 and fromUTF8. +// +template<size_t wcharSize> +struct WstringHelper { - string result; - result.reserve(str.length() * 2); + static ConversionResult toUTF8( + const wchar_t*& sourceStart, const wchar_t* sourceEnd, + Byte*& targetStart, Byte* targetEnd, ConversionFlags flags); - for(unsigned int i = 0; i < str.length(); ++i) + static ConversionResult fromUTF8( + const Byte*& sourceStart, const Byte* sourceEnd, + wchar_t*& targetStart, wchar_t* targetEnd, ConversionFlags flags); +}; + +template<> +struct WstringHelper<2> +{ + static ConversionResult toUTF8( + const wchar_t*& sourceStart, const wchar_t* sourceEnd, + Byte*& targetStart, Byte* targetEnd, ConversionFlags flags) { - wchar_t wc; - wc = str[i]; - - if(wc < 0x80) - { - result += static_cast<char>(wc); - } - else if(wc < 0x800) - { - result += 0xc0 | (wc>>6); - result += 0x80 | (wc & 0x3f); - } - else if(wc < 0x10000) - { - result += 0xe0 | (wc>>12); - result += 0x80 | ((wc>>6) & 0x3f); - result += 0x80 | (wc & 0x3f); - } -#if SIZEOF_WCHAR_T >= 4 - else if(wc < 0x10FFFF) - { - result += 0xf0 | (wc>>18); - result += 0x80 | ((wc>>12) & 0x3f); - result += 0x80 | ((wc>>6) & 0x3f); - result += 0x80 | (wc & 0x3f); - } -#endif - else - { - return result; // Error, not encodable. - } + return ConvertUTF16toUTF8( + reinterpret_cast<const UTF16**>(&sourceStart), + reinterpret_cast<const UTF16*>(sourceEnd), + &targetStart, targetEnd, flags); } + + static ConversionResult fromUTF8( + const Byte*& sourceStart, const Byte* sourceEnd, + wchar_t*& targetStart, wchar_t* targetEnd, ConversionFlags flags) + { + return ConvertUTF8toUTF16( + &sourceStart, sourceEnd, + reinterpret_cast<UTF16**>(&targetStart), + reinterpret_cast<UTF16*>(targetEnd), flags); + } +}; - return result; +template<> +struct WstringHelper<4> +{ + static ConversionResult toUTF8( + const wchar_t*& sourceStart, const wchar_t* sourceEnd, + Byte*& targetStart, Byte* targetEnd, ConversionFlags flags) + { + return ConvertUTF32toUTF8( + reinterpret_cast<const UTF32**>(&sourceStart), + reinterpret_cast<const UTF32*>(sourceEnd), + &targetStart, targetEnd, flags); + } + + static ConversionResult fromUTF8( + const Byte*& sourceStart, const Byte* sourceEnd, + wchar_t*& targetStart, wchar_t* targetEnd, ConversionFlags flags) + { + return ConvertUTF8toUTF32( + &sourceStart, sourceEnd, + reinterpret_cast<UTF32**>(&targetStart), + reinterpret_cast<UTF32*>(targetEnd), flags); + } +}; } -wstring -IceUtil::stringToWstring(const string& str) +// +// convertXXX functions +// + +ConversionResult +IceUtil::convertUTFWstringToUTF8( + const wchar_t*& sourceStart, const wchar_t* sourceEnd, + Byte*& targetStart, Byte* targetEnd, ConversionFlags flags) { - wstring result; - result.reserve(str.length()); + return WstringHelper<sizeof(wchar_t)>::toUTF8( + sourceStart, sourceEnd, targetStart, targetEnd, flags); +} + +ConversionResult +IceUtil::convertUTF8ToUTFWstring( + const Byte*& sourceStart, const Byte* sourceEnd, + wchar_t*& targetStart, wchar_t* targetEnd, ConversionFlags flags) +{ + return WstringHelper<sizeof(wchar_t)>::fromUTF8( + sourceStart, sourceEnd, targetStart, targetEnd, flags); +} - unsigned int len; - for(unsigned int i = 0; i < str.length(); i += len) +ConversionResult +IceUtil::convertUTF8ToUTFWstring(const Byte*& sourceStart, const Byte* sourceEnd, + std::wstring& target, ConversionFlags flags) +{ + // + // Could be reimplemented without this temporary wchar_t buffer + // + size_t size = static_cast<size_t>(sourceEnd - sourceStart); + wchar_t* outBuf = new wchar_t[size]; + wchar_t* targetStart = outBuf; + wchar_t* targetEnd = targetStart + size; + + ConversionResult result = + convertUTF8ToUTFWstring(sourceStart, sourceEnd, targetStart, + targetEnd, flags); + + if(result == conversionOK) { - unsigned char c = str[i]; - wchar_t wc; - int minval; - - if(c < 0x80) - { - wc = c; - len = 1; - minval = 0; - } - else if(c < 0xc0) // Lead byte must not be 10xxxxxx - { - return result; // Error, not encodable. - } - else if(c < 0xe0) // 110xxxxx - { - wc = c & 0x1f; - len = 2; - minval = 0x80; - } - else if(c < 0xf0) // 1110xxxx - { - wc = c & 0xf; - len = 3; - minval = 0x800; - } -#if SIZEOF_WCHAR_T >= 4 - else if(c < 0xf8) // 11110xxx - { - wc = c & 7; - len = 4; - minval = 0x10000; - } - else if(c < 0xfc) // 111110xx - { - // Length 5 and 6 is declared invalid in Unicode 3.1 and ISO 10646:2003. - wc = c & 3; - len = 5; - minval = 0x110000; - } - else if(c < 0xfe) // 1111110x - { - // Length 5 and 6 is declared invalid in Unicode 3.1 and ISO 10646:2003. - wc = c & 1; - len = 6; - minval = 0x4000000; - } -#endif - else - { - return result; // Error, not encodable. - } - - if(i + len - 1 < str.length()) - { - for(unsigned int j = 1; j < len; ++j) - { - if((str[i + j] & 0xc0) != 0x80) // All other bytes must be 10xxxxxx - { - return result; // Error, not encodable. - } - - wc <<= 6; - wc |= str[i + j] & 0x3f; - } - - if(wc < minval) - { - return result; // Error, non-shortest form. - } - else - { - result += wc; - } - } - else - { - return result; // Error, not encodable. - } + std::wstring s(outBuf, static_cast<size_t>(targetStart - outBuf)); + s.swap(target); + } + delete[] outBuf; + return result; +} + + +// +// wstringToString and stringToWstring +// + +string +IceUtil::wstringToString(const wstring& wstr) +{ + string target; + + size_t size = wstr.size() * 3 * (sizeof(wchar_t) / 2); + + Byte* outBuf = new Byte[size]; + Byte* targetStart = outBuf; + Byte* targetEnd = outBuf + size; + + const wchar_t* sourceStart = wstr.data(); + + ConversionResult result = + convertUTFWstringToUTF8( + sourceStart, sourceStart + wstr.size(), + targetStart, targetEnd, lenientConversion); + + if(result == conversionOK) + { + string s(reinterpret_cast<char*>(outBuf), + static_cast<size_t>(targetStart - outBuf)); + s.swap(target); } + delete[] outBuf; + return target; +} +wstring +IceUtil::stringToWstring(const string& str) +{ + wstring result; + const Byte* sourceStart = reinterpret_cast<const Byte*>(str.data()); + + convertUTF8ToUTFWstring(sourceStart, sourceStart + str.size(), + result, lenientConversion); + // + // TODO: check the ConversionResult and do something with it! + // return result; } @@ -166,28 +184,24 @@ IceUtil::stringToWstring(const string& str) string IceUtil::wstringToString(const basic_string<__wchar_t>& str) { - assert(sizeof(__wchar_t) == SIZEOF_WCHAR_T); return wstringToString(*reinterpret_cast<const wstring*>(&str)); } basic_string<__wchar_t> IceUtil::stringToNativeWstring(const string& str) { - assert(sizeof(__wchar_t) == SIZEOF_WCHAR_T); return reinterpret_cast<basic_string<__wchar_t>& >(stringToWstring(str)); } # else string IceUtil::wstringToString(const basic_string<unsigned short>& str) { - assert(sizeof(__wchar_t) == SIZEOF_WCHAR_T); return wstringToString(*reinterpret_cast<const wstring*>(&str)); } basic_string<unsigned short> IceUtil::stringToTypedefWstring(const string& str) { - assert(sizeof(__wchar_t) == SIZEOF_WCHAR_T); return reinterpret_cast<basic_string<unsigned short>& >(stringToWstring(str)); } |