Merge branch 'utf8-text-stream'

Really fix reading from UTF-8 text streams.

Closes #14720.

See https://github.com/wxWidgets/wxWidgets/pull/1304

src/common/convauto.cpp

@@ -309,6 +309,13 @@ wxConvAuto::ToWChar(wchar_t *dst, size_t dstLen,
     size_t rc = m_conv->ToWChar(dst, dstLen, src, srcLen);
     if ( rc == wxCONV_FAILED && m_bomType == wxBOM_None )
     {
+        // we may need more bytes before we can decode the input, don't switch
+        // to the fall-back conversion in this case as it would prevent us from
+        // decoding UTF-8 input when fed it byte by byte, as done by
+        // wxTextInputStream, for example
+        if ( srcLen < m_conv->GetMaxCharLen() )
+            return wxCONV_FAILED;
+
         // if the conversion failed but we didn't really detect anything and
         // simply tried UTF-8 by default, retry it using the fall-back
         if ( m_encDefault != wxFONTENCODING_MAX )

src/common/txtstrm.cpp

@@ -303,12 +303,24 @@ wxString wxTextInputStream::ReadLine()
 {
     wxString line;
 
-    while ( !m_input.Eof() )
+    for ( ;; )
     {
         wxChar c = GetChar();
-        if (!c)
+        if ( m_input.Eof() )
             break;
 
+        if (!c)
+        {
+            // If we failed to get a character and the stream is not at EOF, it
+            // can only mean that decoding the stream contents using our
+            // conversion object failed. In this case, we must signal an error
+            // at the stream level, as otherwise the code using this function
+            // would never know that something went wrong and would continue
+            // calling it again and again, resulting in an infinite loop.
+            m_input.Reset(wxSTREAM_READ_ERROR);
+            break;
+        }
+
         if (EatEOL(c))
             break;
 

src/common/utilscmn.cpp

@@ -613,14 +613,7 @@ static bool ReadAll(wxInputStream *is, wxArrayString& output)
     // the stream could be already at EOF or in wxSTREAM_BROKEN_PIPE state
     is->Reset();
 
-    // Notice that wxTextInputStream doesn't work correctly with wxConvAuto
-    // currently, see #14720, so use the current locale conversion explicitly
-    // under assumption that any external program should be using it too.
-    wxTextInputStream tis(*is, " \t"
-#if wxUSE_UNICODE
-                                    , wxConvLibc
-#endif
-                                                );
+    wxTextInputStream tis(*is);
 
     for ( ;; )
     {