///////////////////////////////////////////////////////////////////////////// // Name: strconv.cpp // Purpose: Unicode conversion classes // Author: Ove Kaaven, Robert Roebling, Vadim Zeitlin, Vaclav Slavik // Modified by: // Created: 29/01/98 // RCS-ID: $Id$ // Copyright: (c) 1999 Ove Kaaven, Robert Roebling, Vaclav Slavik // (c) 2000-2003 Vadim Zeitlin // Licence: wxWindows licence ///////////////////////////////////////////////////////////////////////////// // ============================================================================ // declarations // ============================================================================ // ---------------------------------------------------------------------------- // headers // ---------------------------------------------------------------------------- #if defined(__GNUG__) && !defined(NO_GCC_PRAGMA) #pragma implementation "strconv.h" #endif // For compilers that support precompilation, includes "wx.h". #include "wx/wxprec.h" #ifdef __BORLANDC__ #pragma hdrstop #endif #ifndef WX_PRECOMP #include "wx/intl.h" #include "wx/log.h" #endif // WX_PRECOMP #include "wx/strconv.h" #if wxUSE_WCHAR_T #ifdef __WXMSW__ #include "wx/msw/private.h" #endif #ifdef __WINDOWS__ #include "wx/msw/missing.h" #endif #ifndef __WXWINCE__ #include #endif #include #include #include #if defined(__WIN32__) && !defined(__WXMICROWIN__) #define wxHAVE_WIN32_MB2WC #endif // __WIN32__ but !__WXMICROWIN__ // ---------------------------------------------------------------------------- // headers // ---------------------------------------------------------------------------- #ifdef __SALFORDC__ #include #endif #ifdef HAVE_ICONV #include #endif #include "wx/encconv.h" #include "wx/fontmap.h" #include "wx/utils.h" #ifdef __WXMAC__ #include #include #include #include "wx/mac/private.h" // includes mac headers #endif // ---------------------------------------------------------------------------- // macros // ---------------------------------------------------------------------------- #define BSWAP_UCS4(str, len) { unsigned _c; for (_c=0; _c=0x110000) { return (size_t)-1; } else { if (output) { *output++ = (wxUint16) ((input >> 10)+0xd7c0); *output = (wxUint16) ((input&0x3ff)+0xdc00); } return 2; } } static size_t decode_utf16(const wxUint16* input, wxUint32& output) { if ((*input<0xd800) || (*input>0xdfff)) { output = *input; return 1; } else if ((input[1]<0xdc00) || (input[1]>=0xdfff)) { output = *input; return (size_t)-1; } else { output = ((input[0] - 0xd7c0) << 10) + (input[1] - 0xdc00); return 2; } } // ---------------------------------------------------------------------------- // wxMBConv // ---------------------------------------------------------------------------- wxMBConv::~wxMBConv() { // nothing to do here } const wxWCharBuffer wxMBConv::cMB2WC(const char *psz) const { if ( psz ) { // calculate the length of the buffer needed first size_t nLen = MB2WC(NULL, psz, 0); if ( nLen != (size_t)-1 ) { // now do the actual conversion wxWCharBuffer buf(nLen); nLen = MB2WC(buf.data(), psz, nLen + 1); // with the trailing NULL if ( nLen != (size_t)-1 ) { return buf; } } } wxWCharBuffer buf((wchar_t *)NULL); return buf; } const wxCharBuffer wxMBConv::cWC2MB(const wchar_t *pwz) const { if ( pwz ) { size_t nLen = WC2MB(NULL, pwz, 0); if ( nLen != (size_t)-1 ) { wxCharBuffer buf(nLen+3); // space for a wxUint32 trailing zero nLen = WC2MB(buf.data(), pwz, nLen + 4); if ( nLen != (size_t)-1 ) { return buf; } } } wxCharBuffer buf((char *)NULL); return buf; } // ---------------------------------------------------------------------------- // wxMBConvLibc // ---------------------------------------------------------------------------- size_t wxMBConvLibc::MB2WC(wchar_t *buf, const char *psz, size_t n) const { return wxMB2WC(buf, psz, n); } size_t wxMBConvLibc::WC2MB(char *buf, const wchar_t *psz, size_t n) const { return wxWC2MB(buf, psz, n); } // ---------------------------------------------------------------------------- // UTF-7 // ---------------------------------------------------------------------------- #if 0 static char utf7_setD[]="ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz" "0123456789'(),-./:?"; static char utf7_setO[]="!\"#$%&*;<=>@[]^_`{|}"; static char utf7_setB[]="ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz" "0123456789+/"; #endif // TODO: write actual implementations of UTF-7 here size_t wxMBConvUTF7::MB2WC(wchar_t * WXUNUSED(buf), const char * WXUNUSED(psz), size_t WXUNUSED(n)) const { return 0; } size_t wxMBConvUTF7::WC2MB(char * WXUNUSED(buf), const wchar_t * WXUNUSED(psz), size_t WXUNUSED(n)) const { return 0; } // ---------------------------------------------------------------------------- // UTF-8 // ---------------------------------------------------------------------------- static wxUint32 utf8_max[]= { 0x7f, 0x7ff, 0xffff, 0x1fffff, 0x3ffffff, 0x7fffffff, 0xffffffff }; size_t wxMBConvUTF8::MB2WC(wchar_t *buf, const char *psz, size_t n) const { size_t len = 0; while (*psz && ((!buf) || (len < n))) { unsigned char cc = *psz++, fc = cc; unsigned cnt; for (cnt = 0; fc & 0x80; cnt++) fc <<= 1; if (!cnt) { // plain ASCII char if (buf) *buf++ = cc; len++; } else { cnt--; if (!cnt) { // invalid UTF-8 sequence return (size_t)-1; } else { unsigned ocnt = cnt - 1; wxUint32 res = cc & (0x3f >> cnt); while (cnt--) { cc = *psz++; if ((cc & 0xC0) != 0x80) { // invalid UTF-8 sequence return (size_t)-1; } res = (res << 6) | (cc & 0x3f); } if (res <= utf8_max[ocnt]) { // illegal UTF-8 encoding return (size_t)-1; } #ifdef WC_UTF16 // cast is ok because wchar_t == wxUuint16 if WC_UTF16 size_t pa = encode_utf16(res, (wxUint16 *)buf); if (pa == (size_t)-1) return (size_t)-1; if (buf) buf += pa; len += pa; #else // !WC_UTF16 if (buf) *buf++ = res; len++; #endif // WC_UTF16/!WC_UTF16 } } } if (buf && (len < n)) *buf = 0; return len; } size_t wxMBConvUTF8::WC2MB(char *buf, const wchar_t *psz, size_t n) const { size_t len = 0; while (*psz && ((!buf) || (len < n))) { wxUint32 cc; #ifdef WC_UTF16 // cast is ok for WC_UTF16 size_t pa = decode_utf16((const wxUint16 *)psz, cc); psz += (pa == (size_t)-1) ? 1 : pa; #else cc=(*psz++) & 0x7fffffff; #endif unsigned cnt; for (cnt = 0; cc > utf8_max[cnt]; cnt++) {} if (!cnt) { // plain ASCII char if (buf) *buf++ = (char) cc; len++; } else { len += cnt + 1; if (buf) { *buf++ = (char) ((-128 >> cnt) | ((cc >> (cnt * 6)) & (0x3f >> cnt))); while (cnt--) *buf++ = (char) (0x80 | ((cc >> (cnt * 6)) & 0x3f)); } } } if (buf && (len 1) { *(wxUint16*)buf = cc[1]; buf += sizeof(wxUint16); } } len += pa*sizeof(wxUint16); psz++; } if (buf && len<=n-sizeof(wxUint16)) *(wxUint16*)buf=0; return len; } // swap 16bit MB to 32bit String size_t wxMBConvUTF16swap::MB2WC(wchar_t *buf, const char *psz, size_t n) const { size_t len=0; while (*(wxUint16*)psz && (!buf || len < n)) { wxUint32 cc; char tmp[4]; tmp[0]=psz[1]; tmp[1]=psz[0]; tmp[2]=psz[3]; tmp[3]=psz[2]; size_t pa=decode_utf16((wxUint16*)tmp, cc); if (pa == (size_t)-1) return pa; if (buf) *buf++ = cc; len++; psz += pa * sizeof(wxUint16); } if (buf && len 1) { *buf++ = ((char*)cc)[3]; *buf++ = ((char*)cc)[2]; } } len += pa*sizeof(wxUint16); psz++; } if (buf && len<=n-sizeof(wxUint16)) *(wxUint16*)buf=0; return len; } #endif // WC_UTF16 // ---------------------------------------------------------------------------- // UTF-32 // ---------------------------------------------------------------------------- #ifdef WORDS_BIGENDIAN #define wxMBConvUTF32straight wxMBConvUTF32BE #define wxMBConvUTF32swap wxMBConvUTF32LE #else #define wxMBConvUTF32swap wxMBConvUTF32BE #define wxMBConvUTF32straight wxMBConvUTF32LE #endif WXDLLIMPEXP_DATA_BASE(wxMBConvUTF32LE) wxConvUTF32LE; WXDLLIMPEXP_DATA_BASE(wxMBConvUTF32BE) wxConvUTF32BE; #ifdef WC_UTF16 // copy 32bit MB to 16bit String size_t wxMBConvUTF32straight::MB2WC(wchar_t *buf, const char *psz, size_t n) const { size_t len=0; while (*(wxUint32*)psz && (!buf || len < n)) { wxUint16 cc[2]; size_t pa=encode_utf16(*(wxUint32*)psz, cc); if (pa == (size_t)-1) return pa; if (buf) { *buf++ = cc[0]; if (pa > 1) *buf++ = cc[1]; } len += pa; psz += sizeof(wxUint32); } if (buf && len 1) *buf++ = cc[1]; } len += pa; psz += sizeof(wxUint32); } if (buf && len= 4 && verMin >= 10; break; case wxWINDOWS_NT: s_isWin98Or2k = verMaj >= 5; break; default: // unknown, be conseravtive by default s_isWin98Or2k = 0; } wxASSERT_MSG( s_isWin98Or2k != -1, _T("should be set above") ); } return s_isWin98Or2k == 1; } long m_CodePage; }; #endif // wxHAVE_WIN32_MB2WC // ============================================================================ // Mac conversion classes // ============================================================================ #if defined(__WXMAC__) && defined(TARGET_CARBON) class wxMBConv_mac : public wxMBConv { public: wxMBConv_mac() { Init(CFStringGetSystemEncoding()) ; } wxMBConv_mac(const wxChar* name) { Init( wxMacGetSystemEncFromFontEnc(wxFontMapper::Get()->CharsetToEncoding(name, FALSE) ) ) ; } wxMBConv_mac(wxFontEncoding encoding) { Init( wxMacGetSystemEncFromFontEnc(encoding) ); } ~wxMBConv_mac() { OSStatus status = noErr ; status = TECDisposeConverter(m_MB2WC_converter); status = TECDisposeConverter(m_WC2MB_converter); } void Init( TextEncodingBase encoding) { OSStatus status = noErr ; m_char_encoding = encoding ; m_unicode_encoding = CreateTextEncoding(kTextEncodingUnicodeDefault,0,kUnicode16BitFormat) ; status = TECCreateConverter(&m_MB2WC_converter, m_char_encoding, m_unicode_encoding); status = TECCreateConverter(&m_WC2MB_converter, m_unicode_encoding, m_char_encoding); } size_t MB2WC(wchar_t *buf, const char *psz, size_t n) const { OSStatus status = noErr ; ByteCount byteOutLen ; ByteCount byteInLen = strlen(psz) ; wchar_t *tbuf = NULL ; UniChar* ubuf = NULL ; size_t res = 0 ; if (buf == NULL) { n = byteInLen ; tbuf = (wchar_t*) malloc( n * SIZEOF_WCHAR_T) ; } ByteCount byteBufferLen = n * sizeof( UniChar ) ; #if SIZEOF_WCHAR_T == 4 ubuf = (UniChar*) malloc( byteBufferLen + 2 ) ; #else ubuf = (UniChar*) (buf ? buf : tbuf) ; #endif status = TECConvertText(m_MB2WC_converter, (ConstTextPtr) psz , byteInLen, &byteInLen, (TextPtr) ubuf , byteBufferLen, &byteOutLen); #if SIZEOF_WCHAR_T == 4 // we have to terminate here, because n might be larger for the trailing zero, and if UniChar // is not properly terminated we get random characters at the end ubuf[byteOutLen / sizeof( UniChar ) ] = 0 ; wxMBConvUTF16BE converter ; res = converter.MB2WC( (buf ? buf : tbuf) , (const char*)ubuf , n ) ; free( ubuf ) ; #else res = byteOutLen / sizeof( UniChar ) ; #endif if ( buf == NULL ) free(tbuf) ; if ( buf && res < n) buf[res] = 0; return res ; } size_t WC2MB(char *buf, const wchar_t *psz, size_t n) const { OSStatus status = noErr ; ByteCount byteOutLen ; ByteCount byteInLen = wxWcslen(psz) * SIZEOF_WCHAR_T ; char *tbuf = NULL ; if (buf == NULL) { // worst case n = byteInLen * 2 ; tbuf = (char*) malloc( n ) ; } ByteCount byteBufferLen = n ; UniChar* ubuf = NULL ; #if SIZEOF_WCHAR_T == 4 wxMBConvUTF16BE converter ; size_t unicharlen = converter.WC2MB( NULL , psz , 0 ) ; byteInLen = unicharlen ; ubuf = (UniChar*) malloc( byteInLen + 2 ) ; converter.WC2MB( (char*) ubuf , psz, unicharlen + 2 ) ; #else ubuf = (UniChar*) psz ; #endif status = TECConvertText(m_WC2MB_converter, (ConstTextPtr) ubuf , byteInLen, &byteInLen, (TextPtr) (buf ? buf : tbuf) , byteBufferLen, &byteOutLen); #if SIZEOF_WCHAR_T == 4 free( ubuf ) ; #endif if ( buf == NULL ) free(tbuf) ; size_t res = byteOutLen ; if ( buf && res < n) buf[res] = 0; return res ; } bool IsOk() const { return m_MB2WC_converter != NULL && m_WC2MB_converter != NULL ; } private: TECObjectRef m_MB2WC_converter ; TECObjectRef m_WC2MB_converter ; TextEncodingBase m_char_encoding ; TextEncodingBase m_unicode_encoding ; }; #endif // defined(__WXMAC__) && defined(TARGET_CARBON) // ============================================================================ // wxEncodingConverter based conversion classes // ============================================================================ #if wxUSE_FONTMAP class wxMBConv_wxwin : public wxMBConv { private: void Init() { m_ok = m2w.Init(m_enc, wxFONTENCODING_UNICODE) && w2m.Init(wxFONTENCODING_UNICODE, m_enc); } public: // temporarily just use wxEncodingConverter stuff, // so that it works while a better implementation is built wxMBConv_wxwin(const wxChar* name) { if (name) m_enc = wxFontMapper::Get()->CharsetToEncoding(name, false); else m_enc = wxFONTENCODING_SYSTEM; Init(); } wxMBConv_wxwin(wxFontEncoding enc) { m_enc = enc; Init(); } size_t MB2WC(wchar_t *buf, const char *psz, size_t WXUNUSED(n)) const { size_t inbuf = strlen(psz); if (buf) m2w.Convert(psz,buf); return inbuf; } size_t WC2MB(char *buf, const wchar_t *psz, size_t WXUNUSED(n)) const { const size_t inbuf = wxWcslen(psz); if (buf) w2m.Convert(psz,buf); return inbuf; } bool IsOk() const { return m_ok; } public: wxFontEncoding m_enc; wxEncodingConverter m2w, w2m; // were we initialized successfully? bool m_ok; DECLARE_NO_COPY_CLASS(wxMBConv_wxwin) }; #endif // wxUSE_FONTMAP // ============================================================================ // wxCSConv implementation // ============================================================================ void wxCSConv::Init() { m_name = NULL; m_convReal = NULL; m_deferred = true; } wxCSConv::wxCSConv(const wxChar *charset) { Init(); if ( charset ) { SetName(charset); } m_encoding = wxFONTENCODING_SYSTEM; } wxCSConv::wxCSConv(wxFontEncoding encoding) { if ( encoding == wxFONTENCODING_MAX || encoding == wxFONTENCODING_DEFAULT ) { wxFAIL_MSG( _T("invalid encoding value in wxCSConv ctor") ); encoding = wxFONTENCODING_SYSTEM; } Init(); m_encoding = encoding; } wxCSConv::~wxCSConv() { Clear(); } wxCSConv::wxCSConv(const wxCSConv& conv) : wxMBConv() { Init(); SetName(conv.m_name); m_encoding = conv.m_encoding; } wxCSConv& wxCSConv::operator=(const wxCSConv& conv) { Clear(); SetName(conv.m_name); m_encoding = conv.m_encoding; return *this; } void wxCSConv::Clear() { free(m_name); delete m_convReal; m_name = NULL; m_convReal = NULL; } void wxCSConv::SetName(const wxChar *charset) { if (charset) { m_name = wxStrdup(charset); m_deferred = true; } } wxMBConv *wxCSConv::DoCreate() const { // check for the special case of ASCII or ISO8859-1 charset: as we have // special knowledge of it anyhow, we don't need to create a special // conversion object if ( m_encoding == wxFONTENCODING_ISO8859_1 ) { // don't convert at all return NULL; } // we trust OS to do conversion better than we can so try external // conversion methods first // // the full order is: // 1. OS conversion (iconv() under Unix or Win32 API) // 2. hard coded conversions for UTF // 3. wxEncodingConverter as fall back // step (1) #ifdef HAVE_ICONV #if !wxUSE_FONTMAP if ( m_name ) #endif // !wxUSE_FONTMAP { wxString name(m_name); #if wxUSE_FONTMAP if ( name.empty() ) name = wxFontMapper::Get()->GetEncodingName(m_encoding); #endif // wxUSE_FONTMAP wxMBConv_iconv *conv = new wxMBConv_iconv(name); if ( conv->IsOk() ) return conv; delete conv; } #endif // HAVE_ICONV #ifdef wxHAVE_WIN32_MB2WC { #if wxUSE_FONTMAP wxMBConv_win32 *conv = m_name ? new wxMBConv_win32(m_name) : new wxMBConv_win32(m_encoding); if ( conv->IsOk() ) return conv; delete conv; #else return NULL; #endif } #endif // wxHAVE_WIN32_MB2WC #if defined(__WXMAC__) { if ( m_name || ( m_encoding < wxFONTENCODING_UTF16BE ) ) { wxMBConv_mac *conv = m_name ? new wxMBConv_mac(m_name) : new wxMBConv_mac(m_encoding); if ( conv->IsOk() ) return conv; delete conv; } } #endif // step (2) wxFontEncoding enc = m_encoding; #if wxUSE_FONTMAP if ( enc == wxFONTENCODING_SYSTEM && m_name ) { // use "false" to suppress interactive dialogs -- we can be called from // anywhere and popping up a dialog from here is the last thing we want to // do enc = wxFontMapper::Get()->CharsetToEncoding(m_name, false); } #endif // wxUSE_FONTMAP switch ( enc ) { case wxFONTENCODING_UTF7: return new wxMBConvUTF7; case wxFONTENCODING_UTF8: return new wxMBConvUTF8; case wxFONTENCODING_UTF16BE: return new wxMBConvUTF16BE; case wxFONTENCODING_UTF16LE: return new wxMBConvUTF16LE; case wxFONTENCODING_UTF32BE: return new wxMBConvUTF32BE; case wxFONTENCODING_UTF32LE: return new wxMBConvUTF32LE; default: // nothing to do but put here to suppress gcc warnings ; } // step (3) #if wxUSE_FONTMAP { wxMBConv_wxwin *conv = m_name ? new wxMBConv_wxwin(m_name) : new wxMBConv_wxwin(m_encoding); if ( conv->IsOk() ) return conv; delete conv; } #endif // wxUSE_FONTMAP // NB: This is a hack to prevent deadlock. What could otherwise happen // in Unicode build: wxConvLocal creation ends up being here // because of some failure and logs the error. But wxLog will try to // attach timestamp, for which it will need wxConvLocal (to convert // time to char* and then wchar_t*), but that fails, tries to log // error, but wxLog has a (already locked) critical section that // guards static buffer. static bool alreadyLoggingError = false; if (!alreadyLoggingError) { alreadyLoggingError = true; wxLogError(_("Cannot convert from the charset '%s'!"), m_name ? m_name : #if wxUSE_FONTMAP wxFontMapper::GetEncodingDescription(m_encoding).c_str() #else // !wxUSE_FONTMAP wxString::Format(_("encoding %s"), m_encoding).c_str() #endif // wxUSE_FONTMAP/!wxUSE_FONTMAP ); alreadyLoggingError = false; } return NULL; } void wxCSConv::CreateConvIfNeeded() const { if ( m_deferred ) { wxCSConv *self = (wxCSConv *)this; // const_cast #if wxUSE_INTL // if we don't have neither the name nor the encoding, use the default // encoding for this system if ( !m_name && m_encoding == wxFONTENCODING_SYSTEM ) { self->m_name = wxStrdup(wxLocale::GetSystemEncodingName()); } #endif // wxUSE_INTL self->m_convReal = DoCreate(); self->m_deferred = false; } } size_t wxCSConv::MB2WC(wchar_t *buf, const char *psz, size_t n) const { CreateConvIfNeeded(); if (m_convReal) return m_convReal->MB2WC(buf, psz, n); // latin-1 (direct) size_t len = strlen(psz); if (buf) { for (size_t c = 0; c <= len; c++) buf[c] = (unsigned char)(psz[c]); } return len; } size_t wxCSConv::WC2MB(char *buf, const wchar_t *psz, size_t n) const { CreateConvIfNeeded(); if (m_convReal) return m_convReal->WC2MB(buf, psz, n); // latin-1 (direct) const size_t len = wxWcslen(psz); if (buf) { for (size_t c = 0; c <= len; c++) { if (psz[c] > 0xFF) return (size_t)-1; buf[c] = psz[c]; } } else { for (size_t c = 0; c <= len; c++) { if (psz[c] > 0xFF) return (size_t)-1; } } return len; } // ---------------------------------------------------------------------------- // globals // ---------------------------------------------------------------------------- #ifdef __WINDOWS__ static wxMBConv_win32 wxConvLibcObj; #else static wxMBConvLibc wxConvLibcObj; #endif static wxCSConv wxConvLocalObj(wxFONTENCODING_SYSTEM); static wxCSConv wxConvISO8859_1Obj(wxFONTENCODING_ISO8859_1); static wxMBConvUTF7 wxConvUTF7Obj; static wxMBConvUTF8 wxConvUTF8Obj; WXDLLIMPEXP_DATA_BASE(wxMBConv&) wxConvLibc = wxConvLibcObj; WXDLLIMPEXP_DATA_BASE(wxCSConv&) wxConvLocal = wxConvLocalObj; WXDLLIMPEXP_DATA_BASE(wxCSConv&) wxConvISO8859_1 = wxConvISO8859_1Obj; WXDLLIMPEXP_DATA_BASE(wxMBConvUTF7&) wxConvUTF7 = wxConvUTF7Obj; WXDLLIMPEXP_DATA_BASE(wxMBConvUTF8&) wxConvUTF8 = wxConvUTF8Obj; WXDLLIMPEXP_DATA_BASE(wxMBConv *) wxConvCurrent = &wxConvLibcObj; #else // !wxUSE_WCHAR_T // stand-ins in absence of wchar_t WXDLLIMPEXP_DATA_BASE(wxMBConv) wxConvLibc, wxConvISO8859_1, wxConvLocal, wxConvUTF8; #endif // wxUSE_WCHAR_T/!wxUSE_WCHAR_T