Merge branch 'zip64-fix'

Notably fix ZIP64 creation with individual files larger than 4GB.

See https://github.com/wxWidgets/wxWidgets/pull/730

include/wx/zipstrm.h

@@ -110,6 +110,14 @@ enum wxZipFlags
     wxZIP_RESERVED          = 0xF000
 };
 
+enum wxZipArchiveFormat
+{
+    /// Default zip format
+    wxZIP_FORMAT_DEFAULT,
+    /// ZIP64 format
+    wxZIP_FORMAT_ZIP64
+};
+
 // Forward decls
 //
 class WXDLLIMPEXP_FWD_BASE wxZipEntry;
@@ -131,6 +139,8 @@ public:
 /////////////////////////////////////////////////////////////////////////////
 // Zip Entry - holds the meta data for a file in the zip
 
+class wxDataOutputStream;
+
 class WXDLLIMPEXP_BASE wxZipEntry : public wxArchiveEntry
 {
 public:
@@ -225,7 +235,7 @@ private:
     wxArchiveEntry* DoClone() const wxOVERRIDE             { return ZipClone(); }
 
     size_t ReadLocal(wxInputStream& stream, wxMBConv& conv);
-    size_t WriteLocal(wxOutputStream& stream, wxMBConv& conv) const;
+    size_t WriteLocal(wxOutputStream& stream, wxMBConv& conv, wxZipArchiveFormat zipFormat);
 
     size_t ReadCentral(wxInputStream& stream, wxMBConv& conv);
     size_t WriteCentral(wxOutputStream& stream, wxMBConv& conv) const;
@@ -234,6 +244,9 @@ private:
     size_t WriteDescriptor(wxOutputStream& stream, wxUint32 crc,
                            wxFileOffset compressedSize, wxFileOffset size);
 
+    void WriteLocalFileSizes(wxDataOutputStream& ds) const;
+    void WriteLocalZip64ExtraInfo(wxOutputStream& stream) const;
+
     bool LoadExtraInfo(const char* extraData, wxUint16 extraLen, bool localInfo);
 
     wxUint16 GetInternalFlags(bool checkForUTF8) const;
@@ -256,6 +269,7 @@ private:
     wxUint16     m_DiskStart;          // for multidisk archives, not unsupported
     wxUint16     m_InternalAttributes; // bit 0 set for text files
     wxUint32     m_ExternalAttributes; // system specific depends on SystemMadeBy
+    wxUint16     m_z64infoOffset;      // Offset of ZIP64 local extra data for file sizes
 
     class wxZipMemory *m_Extra;
     class wxZipMemory *m_LocalExtra;
@@ -307,6 +321,9 @@ public:
     int  GetLevel() const                       { return m_level; }
     void WXZIPFIX SetLevel(int level);
 
+    void SetFormat(wxZipArchiveFormat format)   { m_format = format; };
+    wxZipArchiveFormat GetFormat() const        { return m_format; };
+
 protected:
     virtual size_t WXZIPFIX OnSysWrite(const void *buffer, size_t size) wxOVERRIDE;
     virtual wxFileOffset OnSysTell() const wxOVERRIDE      { return m_entrySize; }
@@ -351,6 +368,7 @@ private:
     wxFileOffset m_offsetAdjustment;
     wxString m_Comment;
     bool m_endrecWritten;
+    wxZipArchiveFormat m_format;
 
     wxDECLARE_NO_COPY_CLASS(wxZipOutputStream);
 };

interface/wx/zipstrm.h

@@ -80,6 +80,19 @@ enum wxZipFlags
     wxZIP_RESERVED          = 0xF000
 };
 
+/**
+    Zip archive format
+
+    @since 3.1.1
+*/
+enum wxZipArchiveFormat
+{
+    /// Default zip format: use ZIP64 if it is determined to be necessary.
+    wxZIP_FORMAT_DEFAULT,
+    /// ZIP64 format: force the use of ZIP64 format.
+    wxZIP_FORMAT_ZIP64
+};
+
 
 /**
     @class wxZipNotifier
@@ -558,11 +571,17 @@ public:
     //@{
     /**
         Takes ownership of @a entry and uses it to create a new entry in the zip.
+
+        If you do not specify a size and plan to put more than 4GB data into the
+        entry see SetFormat()
     */
     bool PutNextEntry(wxZipEntry* entry);
 
     /**
         Create a new entry with the given name, timestamp and size.
+
+        If you do not specify a size and plan to put more than 4GB data into the
+        entry see SetFormat()
     */
     bool PutNextEntry(const wxString& name,
                       const wxDateTime& dt = wxDateTime::Now(),
@@ -574,5 +593,33 @@ public:
         It is written at the end of the zip.
     */
     void SetComment(const wxString& comment);
+
+    /**
+        Set the format of the archive.
+
+        The normal zip format is limited to single files and the complete
+        archive smaller than 4GB with less then 65k files. If any of these
+        limits are exceeded, this class will automatically create a ZIP64 file,
+        so in most situations calling SetFormat() is not necessary.
+
+        However to support single entries with more than 4GB of data
+        (compressed or original) whose sizes are unknown when adding the
+        entry with PutNextEntry(), the format has to be set to
+        wxZIP_FORMAT_ZIP64 before adding such entries.
+
+        @since 3.1.1
+    */
+    void SetFormat(wxZipArchiveFormat format);
+
+    /**
+        Get the format of the archive.
+
+        This returns the value passed to SetFormat() and not necessarily the
+        actual archive format (e.g. this method could return
+        wxZIP_FORMAT_DEFAULT even if a ZIP64 will end up being created).
+
+        @since 3.1.1
+
+    wxZipArchiveFormat GetFormat() const;
 };
 

src/common/zipstrm.cpp

@@ -133,7 +133,7 @@ static inline wxUint64 CrackUint64(const char *m)
         (static_cast<wxUint64>(n[6]) << 48) |
         (static_cast<wxUint64>(n[5]) << 40) |
         (static_cast<wxUint64>(n[4]) << 32) |
-        (n[3] << 24) | (n[2] << 16) | (n[1] << 8) | n[0];
+        (static_cast<wxUint64>(n[3]) << 24) | (n[2] << 16) | (n[1] << 8) | n[0];
 }
 
 // Decode a little endian wxUint32 number from a character array
@@ -757,6 +757,7 @@ wxZipEntry::wxZipEntry(
     m_DiskStart(0),
     m_InternalAttributes(0),
     m_ExternalAttributes(0),
+    m_z64infoOffset(0),
     m_Extra(NULL),
     m_LocalExtra(NULL),
     m_zipnotifier(NULL),
@@ -792,6 +793,7 @@ wxZipEntry::wxZipEntry(const wxZipEntry& e)
     m_DiskStart(e.m_DiskStart),
     m_InternalAttributes(e.m_InternalAttributes),
     m_ExternalAttributes(e.m_ExternalAttributes),
+    m_z64infoOffset(0),
     m_Extra(AddRef(e.m_Extra)),
     m_LocalExtra(AddRef(e.m_LocalExtra)),
     m_zipnotifier(NULL),
@@ -1127,7 +1129,7 @@ size_t wxZipEntry::ReadLocal(wxInputStream& stream, wxMBConv& conv)
     return LOCAL_SIZE + nameLen + extraLen;
 }
 
-size_t wxZipEntry::WriteLocal(wxOutputStream& stream, wxMBConv& conv) const
+size_t wxZipEntry::WriteLocal(wxOutputStream& stream, wxMBConv& conv, wxZipArchiveFormat zipFormat)
 {
     wxString unixName = GetName(wxPATH_UNIX);
     const wxWX2MBbuf name_buf = unixName.mb_str(conv);
@@ -1135,9 +1137,11 @@ size_t wxZipEntry::WriteLocal(wxOutputStream& stream, wxMBConv& conv) const
     if (!name) name = "";
     wxUint16 nameLen = wx_truncate_cast(wxUint16, strlen(name));
 
-    bool z64Required = m_CompressedSize > 0xffffffff || m_Size > 0xffffffff;
+    if ( (zipFormat == wxZIP_FORMAT_ZIP64) ||
+        m_CompressedSize >= 0xffffffff || m_Size >= 0xffffffff )
+        m_z64infoOffset = LOCAL_SIZE + nameLen;
     wxUint16 versionNeeded =
-        z64Required ? Z64_VERSION_NEEDED_TO_EXTRACT : int(m_VersionNeeded);
+        (m_z64infoOffset > 0) ? Z64_VERSION_NEEDED_TO_EXTRACT : int(m_VersionNeeded);
 
     wxDataOutputStream ds(stream);
 
@@ -1145,25 +1149,17 @@ size_t wxZipEntry::WriteLocal(wxOutputStream& stream, wxMBConv& conv) const
     ds.Write32(GetDateTime().GetAsDOS());
 
     ds.Write32(m_Crc);
-    ds.Write32(m_CompressedSize != wxInvalidOffset ?
+    WriteLocalFileSizes(ds);
-               LimitUint32(m_CompressedSize) : 0);
-    ds.Write32(m_Size != wxInvalidOffset ?
-               LimitUint32(m_Size) : 0);
 
     ds << nameLen;
     wxUint16 extraLen = wx_truncate_cast(wxUint16, GetLocalExtraLen());
-    if (z64Required)
+    if (m_z64infoOffset)
         extraLen += 20; // tag and 2x64bit file sizes
     ds.Write16(extraLen);
 
     stream.Write(name, nameLen);
-    if (z64Required)
+    if (m_z64infoOffset > 0)
-    {
+        WriteLocalZip64ExtraInfo(stream);
-        ds.Write16(1);  // id
-        ds.Write16(16); // record size
-        ds.Write64(static_cast<wxInt64>(m_CompressedSize));
-        ds.Write64(static_cast<wxInt64>(m_Size));
-    }
     if (GetLocalExtraLen())
         stream.Write(m_LocalExtra->GetData(), GetLocalExtraLen());
 
@@ -1264,7 +1260,7 @@ size_t wxZipEntry::WriteCentral(wxOutputStream& stream, wxMBConv& conv) const
     }
 
     wxUint16 versionNeeded =
-        (z64Required) ? Z64_VERSION_NEEDED_TO_EXTRACT : GetVersionNeeded();
+        (z64Required || m_z64infoOffset) ? Z64_VERSION_NEEDED_TO_EXTRACT : GetVersionNeeded();
 
     wxDataOutputStream ds(stream);
 
@@ -1288,10 +1284,10 @@ size_t wxZipEntry::WriteCentral(wxOutputStream& stream, wxMBConv& conv) const
     {
         ds.Write16(1); // tag
         ds.Write16(z64InfoLen); // record size
-        if (m_CompressedSize > 0xffffffff)
-            ds.Write64(static_cast<wxInt64>(m_CompressedSize));
         if (m_Size > 0xffffffff)
             ds.Write64(static_cast<wxInt64>(m_Size));
+        if (m_CompressedSize > 0xffffffff)
+            ds.Write64(static_cast<wxInt64>(m_CompressedSize));
         if (m_Offset > 0xffffffff)
             ds.Write64(static_cast<wxInt64>(m_Offset));
     }
@@ -1357,12 +1353,40 @@ size_t wxZipEntry::WriteDescriptor(wxOutputStream& stream, wxUint32 crc,
     wxDataOutputStream ds(stream);
 
     ds.Write32(crc);
-    ds.Write32(wx_truncate_cast(wxUint32, compressedSize));
+    WriteLocalFileSizes(ds);
-    ds.Write32(wx_truncate_cast(wxUint32, size));
 
     return SUMS_SIZE;
 }
 
+void wxZipEntry::WriteLocalFileSizes(wxDataOutputStream& ds) const
+{
+    wxUint32 compressedSize;
+    wxUint32 size;
+    if (m_z64infoOffset > 0)
+    {
+        compressedSize = 0xffffffff;
+        size = 0xffffffff;
+    }
+    else
+    {
+        compressedSize = m_CompressedSize != wxInvalidOffset ?
+             wx_truncate_cast(wxUint32, m_CompressedSize) : 0;
+        size = m_Size != wxInvalidOffset ?
+            wx_truncate_cast(wxUint32, m_Size) : 0;
+    }
+    ds.Write32(compressedSize);
+    ds.Write32(size);
+}
+
+void  wxZipEntry::WriteLocalZip64ExtraInfo(wxOutputStream& stream) const
+{
+    wxDataOutputStream ds(stream);
+    ds.Write16(1);  // id
+    ds.Write16(16); // record size
+    ds.Write64(static_cast<wxUint64>(m_Size));
+    ds.Write64(static_cast<wxUint64>(m_CompressedSize));
+}
+
 // Returns the flags as specified or including the UTF-8 flag if filename
 // contains non ASCII characters
 wxUint16 wxZipEntry::GetInternalFlags(bool checkForUTF8) const
@@ -2173,6 +2197,7 @@ void wxZipOutputStream::Init(int level)
     m_level = level;
     m_offsetAdjustment = wxInvalidOffset;
     m_endrecWritten = false;
+    m_format = wxZIP_FORMAT_DEFAULT;
 }
 
 wxZipOutputStream::~wxZipOutputStream()
@@ -2401,7 +2426,7 @@ void wxZipOutputStream::CreatePendingEntry(const void *buffer, size_t size)
         if (spPending->m_CompressedSize != wxInvalidOffset)
             spPending->m_Flags |= wxZIP_SUMS_FOLLOW;
 
-    m_headerSize = spPending->WriteLocal(*m_parent_o_stream, GetConv());
+    m_headerSize = spPending->WriteLocal(*m_parent_o_stream, GetConv(), m_format);
     m_lasterror = m_parent_o_stream->GetLastError();
 
     if (IsOk()) {
@@ -2454,7 +2479,7 @@ void wxZipOutputStream::CreatePendingEntry()
     }
 
     spPending->m_Flags &= ~wxZIP_SUMS_FOLLOW;
-    m_headerSize = spPending->WriteLocal(*m_parent_o_stream, GetConv());
+    m_headerSize = spPending->WriteLocal(*m_parent_o_stream, GetConv(), m_format);
 
     if (m_parent_o_stream->IsOk()) {
         m_entries.push_back(spPending.release());
@@ -2552,6 +2577,12 @@ bool wxZipOutputStream::CloseEntry()
             m_parent_o_stream->SeekO(headerOffset + SUMS_OFFSET);
             entry.WriteDescriptor(*m_parent_o_stream, m_crcAccumulator,
                                   compressedSize, m_entrySize);
+            if (entry.m_z64infoOffset > 0)
+            {
+                // Skip to ZIP64 extended information extra field
+                m_parent_o_stream->SeekO(headerOffset + entry.m_z64infoOffset);
+                entry.WriteLocalZip64ExtraInfo(*m_parent_o_stream);
+            }
             m_parent_o_stream->SeekO(here);
             m_lasterror = m_parent_o_stream->GetLastError();
         } else {
@@ -2565,7 +2596,14 @@ bool wxZipOutputStream::CloseEntry()
     m_store->Close();
     m_raw = false;
 
-    if (IsOk())
+    if (IsOk() && entry.m_z64infoOffset == 0 &&
+        (entry.m_CompressedSize > 0xffffffff || entry.m_Size > 0xffffffff))
+    {
+        wxLogError(_("error writing zip entry '%s': file too large without ZIP64"),
+            entry.GetName().c_str());
+        m_lasterror = wxSTREAM_WRITE_ERROR;
+    }
+    else if (IsOk())
         m_lasterror = m_parent_o_stream->GetLastError();
     else
         wxLogError(_("error writing zip entry '%s': bad crc or length"),