Merge branch 'linux-spec-files'

wxFile and wxTextFile for working with special/non-seekable files.

docs/changes.txt

@@ -106,6 +106,7 @@ All:
 - Add UTF-8 support to wxZipOutputStream (Tobias Taschner).
 - Update all bundled 3rd party libraries to their latest versions.
 - Use unique prefix for all zlib symbols to avoid link conflicts.
+- Make wxFile::ReadAll() work for unseekable files too.
 
 All (GUI):
 

interface/wx/file.h

@@ -95,11 +95,14 @@ public:
     /**
         Returns the length of the file.
 
-        This method may return ::wxInvalidOffset if the length couldn't be
+        Returns ::wxInvalidOffset if the length couldn't be determined.
-        determined or 0 even for non-empty files if the file is not seekable.
 
-        In general, the only way to determine if the file for which this function
+        Please also note that there is @e no guarantee that reading that many
-        returns 0 is really empty or not is to try reading from it.
+        bytes from the file will always succeed. While this is true for regular
+        files (unless the file size has been changed by another process in
+        between Length() and Read() calls), some special files, such as most
+        files under @c /sys or @c /proc directories under Linux, don't actually
+        contain as much data as their size indicates.
     */
     wxFileOffset Length() const;
 
@@ -377,6 +380,8 @@ public:
     /**
         Reads the entire contents of the file into a string.
 
+        Since wxWidgets 3.1.1 this method also works for unseekable files.
+
         @param str
             Non-@NULL pointer to a string to read data into.
         @param conv

src/common/file.cpp

@@ -261,24 +261,71 @@ bool wxFile::ReadAll(wxString *str, const wxMBConv& conv)
 {
     wxCHECK_MSG( str, false, wxS("Output string must be non-NULL") );
 
+    static const ssize_t READSIZE = 4096;
+
+    wxCharBuffer buf;
+
     ssize_t length = Length();
+    if ( length != -1 )
+    {
         wxCHECK_MSG( (wxFileOffset)length == Length(), false, wxT("huge file not supported") );
 
-    wxCharBuffer buf(length);
+        if ( !buf.extend(length) )
+            return false;
+
         char* p = buf.data();
         for ( ;; )
         {
-        static const ssize_t READSIZE = 4096;
-
             ssize_t nread = Read(p, length > READSIZE ? READSIZE : length);
             if ( nread == wxInvalidOffset )
                 return false;
 
-        p += nread;
+            if ( nread == 0 )
-        if ( length <= nread )
+            {
+                // We have reached EOF before reading the entire length of the
+                // file. This can happen for some special files (e.g. those
+                // under /sys on Linux systems) or even for regular files if
+                // another process has truncated the file since we started
+                // reading it, so deal with it gracefully.
+                buf.shrink(p - buf.data());
                 break;
+            }
 
+            p += nread;
             length -= nread;
+
+            if ( !length )
+            {
+                // Notice that we don't keep reading after getting the expected
+                // number of bytes, even though in principle a situation
+                // similar to the one described above, with another process
+                // extending the file since we started to read it, is possible.
+                // But returning just the data that was in the file when we
+                // originally started reading it isn't really wrong in this
+                // case, so keep things simple and just do it like this.
+                break;
+            }
+        }
+    }
+    else // File is not seekable
+    {
+        for ( ;; )
+        {
+            const size_t len = buf.length();
+            if ( !buf.extend(len + READSIZE) )
+                return false;
+
+            ssize_t nread = Read(buf.data() + len, READSIZE);
+            if ( nread == wxInvalidOffset )
+                return false;
+
+            if ( nread < READSIZE )
+            {
+                // We have reached EOF.
+                buf.shrink(len + nread);
+                break;
+            }
+        }
     }
 
     str->assign(buf, conv);
@@ -427,21 +474,6 @@ wxFileOffset wxFile::Length() const
 {
     wxASSERT( IsOpened() );
 
-    // we use a special method for Linux systems where files in sysfs (i.e.
-    // those under /sys typically) return length of 4096 bytes even when
-    // they're much smaller -- this is a problem as it results in errors later
-    // when we try reading 4KB from them
-#ifdef __LINUX__
-    struct stat st;
-    if ( fstat(m_fd, &st) == 0 )
-    {
-        // returning 0 for the special files indicates to the caller that they
-        // are not seekable
-        return st.st_blocks ? st.st_size : 0;
-    }
-    //else: failed to stat, try the normal method
-#endif // __LINUX__
-
     wxFileOffset iRc = Tell();
     if ( iRc != wxInvalidOffset ) {
         wxFileOffset iLen = const_cast<wxFile *>(this)->SeekEnd();

src/common/textfile.cpp

@@ -93,122 +93,13 @@ bool wxTextFile::OnRead(const wxMBConv& conv)
     // file should be opened
     wxASSERT_MSG( m_file.IsOpened(), wxT("can't read closed file") );
 
-    // read the entire file in memory: this is not the most efficient thing to
+    wxString str;
-    // do it but there is no good way to avoid it in Unicode build because if
+    if ( !m_file.ReadAll(&str, conv) )
-    // we read the file block by block we can't convert each block to Unicode
-    // separately (the last multibyte char in the block might be only partially
-    // read and so the conversion would fail) and, as the file contents is kept
-    // in memory by wxTextFile anyhow, it shouldn't be a big problem to read
-    // the file entirely
-    size_t bufSize = 0;
-
-    // number of bytes to (try to) read from disk at once
-    static const size_t BLOCK_SIZE = 4096;
-
-    wxCharBuffer buf;
-
-    // first determine if the file is seekable or not and so whether we can
-    // determine its length in advance
-    wxFileOffset fileLength;
     {
-        wxLogNull logNull;
+        wxLogError(_("Failed to read text file \"%s\"."), GetName());
-        fileLength = m_file.Length();
-    }
-
-    // some non-seekable files under /proc under Linux pretend that they're
-    // seekable but always return 0; others do return an error
-    const bool seekable = fileLength != wxInvalidOffset && fileLength != 0;
-    if ( seekable )
-    {
-        // we know the required length, so set the buffer size in advance
-        bufSize = fileLength;
-        if ( !buf.extend(bufSize) )
-            return false;
-
-        // if the file is seekable, also check that we're at its beginning
-        wxASSERT_MSG( m_file.Tell() == 0, wxT("should be at start of file") );
-
-        char *dst = buf.data();
-        for ( size_t nRemaining = bufSize; nRemaining > 0; )
-        {
-            size_t nToRead = BLOCK_SIZE;
-
-            // the file size could have changed, avoid overflowing the buffer
-            // even if it did
-            if ( nToRead > nRemaining )
-                nToRead = nRemaining;
-
-            ssize_t nRead = m_file.Read(dst, nToRead);
-
-            if ( nRead == wxInvalidOffset )
-            {
-                // read error (error message already given in wxFile::Read)
         return false;
     }
 
-            if ( nRead == 0 )
-            {
-                // this file can't be empty because we checked for this above
-                // so this must be the end of file
-                break;
-            }
-
-            dst += nRead;
-            nRemaining -= nRead;
-        }
-
-        wxASSERT_MSG( dst - buf.data() == (wxFileOffset)bufSize,
-                      wxT("logic error") );
-    }
-    else // file is not seekable
-    {
-        char block[BLOCK_SIZE];
-        for ( ;; )
-        {
-            ssize_t nRead = m_file.Read(block, WXSIZEOF(block));
-
-            if ( nRead == wxInvalidOffset )
-            {
-                // read error (error message already given in wxFile::Read)
-                return false;
-            }
-
-            if ( nRead == 0 )
-            {
-                // if no bytes have been read, presumably this is a
-                // valid-but-empty file
-                if ( bufSize == 0 )
-                    return true;
-
-                // otherwise we've finished reading the file
-                break;
-            }
-
-            // extend the buffer for new data
-            if ( !buf.extend(bufSize + nRead) )
-                return false;
-
-            // and append it to the buffer
-            memcpy(buf.data() + bufSize, block, nRead);
-            bufSize += nRead;
-        }
-    }
-
-    const wxString str(buf, conv, bufSize);
-
-    // there's no risk of this happening in ANSI build
-#if wxUSE_UNICODE
-    if ( bufSize > 4 && str.empty() )
-    {
-        wxLogError(_("Failed to convert file \"%s\" to Unicode."), GetName());
-        return false;
-    }
-#endif // wxUSE_UNICODE
-
-    // we don't need this memory any more
-    buf.reset();
-
-
     // now break the buffer in lines
 
     // the beginning of the current line, changes inside the loop

tests/file/filetest.cpp

@@ -139,4 +139,33 @@ void FileTestCase::TempFile()
     CPPUNIT_ASSERT( wxRemoveFile(wxT("test2")) );
 }
 
+#ifdef __LINUX__
+
+// Check that GetSize() works correctly for special files.
+TEST_CASE("wxFile::Special", "[file][linux][special-file]")
+{
+    // We can't test /proc/kcore here, unlike in the similar
+    // wxFileName::GetSize() test, as wxFile must be able to open it (at least
+    // for reading) and usually we don't have the permissions to do it.
+
+    // This file is not seekable and has 0 size, but can still be read.
+    wxFile fileProc("/proc/diskstats");
+    CHECK( fileProc.IsOpened() );
+
+    wxString s;
+    CHECK( fileProc.ReadAll(&s) );
+    CHECK( !s.empty() );
+
+    // All files in /sys seem to have size of 4KiB currently, even if they
+    // don't have that much data in them.
+    wxFile fileSys("/sys/power/state");
+    CHECK( fileSys.Length() == 4096 );
+    CHECK( fileSys.IsOpened() );
+    CHECK( fileSys.ReadAll(&s) );
+    CHECK( !s.empty() );
+    CHECK( s.length() < 4096 );
+}
+
+#endif // __LINUX__
+
 #endif // wxUSE_FILE

tests/filename/filenametest.cpp

@@ -1049,3 +1049,18 @@ void FileNameTestCase::TestShortcuts()
 }
 
 #endif // __WINDOWS__
+
+#ifdef __LINUX__
+
+// Check that GetSize() works correctly for special files.
+TEST_CASE("wxFileName::GetSizeSpecial", "[filename][linux][special-file]")
+{
+    wxULongLong size = wxFileName::GetSize("/proc/kcore");
+    INFO( "size of /proc/kcore=" << size );
+    CHECK( size > 0 );
+
+    // All files in /sys seem to have size of 4KiB currently.
+    CHECK( wxFileName::GetSize("/sys/power/state") == 4096 );
+}
+
+#endif // __LINUX__

tests/textfile/textfiletest.cpp

@@ -338,5 +338,29 @@ void TextFileTestCase::ReadBig()
                           f[NUM_LINES - 1] );
 }
 
-#endif // wxUSE_TEXTFILE
+#ifdef __LINUX__
 
+// Check if using wxTextFile with special files, whose reported size doesn't
+// correspond to the real amount of data in them, works.
+TEST_CASE("wxTextFile::Special", "[textfile][linux][special-file]")
+{
+    SECTION("/proc")
+    {
+        wxTextFile f;
+        CHECK( f.Open("/proc/diskstats") );
+        CHECK( f.GetLineCount() > 1 );
+    }
+
+    SECTION("/sys")
+    {
+        wxTextFile f;
+        CHECK( f.Open("/sys/power/state") );
+        REQUIRE( f.GetLineCount() == 1 );
+        INFO( "/sys/power/state contains \"" << f[0] << "\"" );
+        CHECK( f[0].find("mem") != wxString::npos );
+    }
+}
+
+#endif // __LINUX__
+
+#endif // wxUSE_TEXTFILE