Fix ZIP64 creation with individual files larger 4GB

If single files larger than 4GB where added the recorded file sizes
in the local file header where not updated correctly. Additionally
recorded file sizes in the central directory where in the wrong order
making it impossible to extract.

To enable adding files with unknown size which might be larger than 4GB
the new method wxZipOutputStream::SetFormat() is added.

Additionally a check has been added in case a file larger 4GB has been
written without ZIP64 info.

include/wx/zipstrm.h

@@ -110,6 +110,14 @@ enum wxZipFlags
     wxZIP_RESERVED          = 0xF000
 };
 
+enum wxZipArchiveFormat
+{
+    /// Default zip format
+    wxZIP_FORMAT_DEFAULT,
+    /// ZIP64 format
+    wxZIP_FORMAT_ZIP64
+};
+
 // Forward decls
 //
 class WXDLLIMPEXP_FWD_BASE wxZipEntry;
@@ -131,6 +139,8 @@ public:
 /////////////////////////////////////////////////////////////////////////////
 // Zip Entry - holds the meta data for a file in the zip
 
+class wxDataOutputStream;
+
 class WXDLLIMPEXP_BASE wxZipEntry : public wxArchiveEntry
 {
 public:
@@ -225,7 +235,7 @@ private:
     wxArchiveEntry* DoClone() const wxOVERRIDE             { return ZipClone(); }
 
     size_t ReadLocal(wxInputStream& stream, wxMBConv& conv);
-    size_t WriteLocal(wxOutputStream& stream, wxMBConv& conv) const;
+    size_t WriteLocal(wxOutputStream& stream, wxMBConv& conv, wxZipArchiveFormat zipFormat);
 
     size_t ReadCentral(wxInputStream& stream, wxMBConv& conv);
     size_t WriteCentral(wxOutputStream& stream, wxMBConv& conv) const;
@@ -234,6 +244,9 @@ private:
     size_t WriteDescriptor(wxOutputStream& stream, wxUint32 crc,
                            wxFileOffset compressedSize, wxFileOffset size);
 
+    void WriteLocalFileSizes(wxDataOutputStream& ds) const;
+    void WriteLocalZip64ExtraInfo(wxOutputStream& stream) const;
+
     bool LoadExtraInfo(const char* extraData, wxUint16 extraLen, bool localInfo);
 
     wxUint16 GetInternalFlags(bool checkForUTF8) const;
@@ -256,6 +269,7 @@ private:
     wxUint16     m_DiskStart;          // for multidisk archives, not unsupported
     wxUint16     m_InternalAttributes; // bit 0 set for text files
     wxUint32     m_ExternalAttributes; // system specific depends on SystemMadeBy
+    wxUint16     m_z64infoOffset;      // Offset of ZIP64 local extra data for file sizes
 
     class wxZipMemory *m_Extra;
     class wxZipMemory *m_LocalExtra;
@@ -307,6 +321,9 @@ public:
     int  GetLevel() const                       { return m_level; }
     void WXZIPFIX SetLevel(int level);
 
+    void SetFormat(wxZipArchiveFormat format)   { m_format = format; };
+    wxZipArchiveFormat GetFormat() const        { return m_format; };
+
 protected:
     virtual size_t WXZIPFIX OnSysWrite(const void *buffer, size_t size) wxOVERRIDE;
     virtual wxFileOffset OnSysTell() const wxOVERRIDE      { return m_entrySize; }
@@ -351,6 +368,7 @@ private:
     wxFileOffset m_offsetAdjustment;
     wxString m_Comment;
     bool m_endrecWritten;
+    wxZipArchiveFormat m_format;
 
     wxDECLARE_NO_COPY_CLASS(wxZipOutputStream);
 };

interface/wx/zipstrm.h

@@ -80,6 +80,19 @@ enum wxZipFlags
     wxZIP_RESERVED          = 0xF000
 };
 
+/**
+    Zip archive format
+
+    @since 3.1.1
+*/
+enum wxZipArchiveFormat
+{
+    /// Default zip format
+    wxZIP_FORMAT_DEFAULT,
+    /// ZIP64 format
+    wxZIP_FORMAT_ZIP64
+};
+
 
 /**
     @class wxZipNotifier
@@ -558,11 +571,17 @@ public:
     //@{
     /**
         Takes ownership of @a entry and uses it to create a new entry in the zip.
+
+        If you do not specify a size and plan to put more than 4GB data into the
+        entry see SetFormat()
     */
     bool PutNextEntry(wxZipEntry* entry);
 
     /**
         Create a new entry with the given name, timestamp and size.
+
+        If you do not specify a size and plan to put more than 4GB data into the
+        entry see SetFormat()
     */
     bool PutNextEntry(const wxString& name,
                       const wxDateTime& dt = wxDateTime::Now(),
@@ -574,5 +593,30 @@ public:
         It is written at the end of the zip.
     */
     void SetComment(const wxString& comment);
+
+    /**
+        Set the format of the archive.
+
+        The normal zip format is limited to single files and the complete
+        archive smaller than 4GB with less then 65k files. If any of these
+        limits are exceed this class will automatically create a ZIP64 file.
+
+        However to support single entries with more than 4GB of data
+        (compressed or original) which sizes are unknown when adding the
+        entry with PutNextEntry() the format has to be set to
+        wxZIP_FORMAT_ZIP64 adding such entries.
+
+        @since 3.1.1
+    */
+    void SetFormat(wxZipArchiveFormat format);
+
+    /**
+        Get the format of the archive.
+
+        @since 3.1.1
+
+        @see SetFormat()
+    */
+    wxZipArchiveFormat GetFormat() const;
 };
 

src/common/zipstrm.cpp

@@ -757,6 +757,7 @@ wxZipEntry::wxZipEntry(
     m_DiskStart(0),
     m_InternalAttributes(0),
     m_ExternalAttributes(0),
+    m_z64infoOffset(0),
     m_Extra(NULL),
     m_LocalExtra(NULL),
     m_zipnotifier(NULL),
@@ -792,6 +793,7 @@ wxZipEntry::wxZipEntry(const wxZipEntry& e)
     m_DiskStart(e.m_DiskStart),
     m_InternalAttributes(e.m_InternalAttributes),
     m_ExternalAttributes(e.m_ExternalAttributes),
+    m_z64infoOffset(0),
     m_Extra(AddRef(e.m_Extra)),
     m_LocalExtra(AddRef(e.m_LocalExtra)),
     m_zipnotifier(NULL),
@@ -1127,7 +1129,7 @@ size_t wxZipEntry::ReadLocal(wxInputStream& stream, wxMBConv& conv)
     return LOCAL_SIZE + nameLen + extraLen;
 }
 
-size_t wxZipEntry::WriteLocal(wxOutputStream& stream, wxMBConv& conv) const
+size_t wxZipEntry::WriteLocal(wxOutputStream& stream, wxMBConv& conv, wxZipArchiveFormat zipFormat)
 {
     wxString unixName = GetName(wxPATH_UNIX);
     const wxWX2MBbuf name_buf = unixName.mb_str(conv);
@@ -1135,9 +1137,11 @@ size_t wxZipEntry::WriteLocal(wxOutputStream& stream, wxMBConv& conv) const
     if (!name) name = "";
     wxUint16 nameLen = wx_truncate_cast(wxUint16, strlen(name));
 
-    bool z64Required = m_CompressedSize > 0xffffffff || m_Size > 0xffffffff;
+    if ( (zipFormat == wxZIP_FORMAT_ZIP64) ||
+        m_CompressedSize >= 0xffffffff || m_Size >= 0xffffffff )
+        m_z64infoOffset = LOCAL_SIZE + nameLen;
     wxUint16 versionNeeded =
-        z64Required ? Z64_VERSION_NEEDED_TO_EXTRACT : int(m_VersionNeeded);
+        (m_z64infoOffset > 0) ? Z64_VERSION_NEEDED_TO_EXTRACT : int(m_VersionNeeded);
 
     wxDataOutputStream ds(stream);
 
@@ -1145,25 +1149,17 @@ size_t wxZipEntry::WriteLocal(wxOutputStream& stream, wxMBConv& conv) const
     ds.Write32(GetDateTime().GetAsDOS());
 
     ds.Write32(m_Crc);
-    ds.Write32(m_CompressedSize != wxInvalidOffset ?
+    WriteLocalFileSizes(ds);
-               LimitUint32(m_CompressedSize) : 0);
-    ds.Write32(m_Size != wxInvalidOffset ?
-               LimitUint32(m_Size) : 0);
 
     ds << nameLen;
     wxUint16 extraLen = wx_truncate_cast(wxUint16, GetLocalExtraLen());
-    if (z64Required)
+    if (m_z64infoOffset)
         extraLen += 20; // tag and 2x64bit file sizes
     ds.Write16(extraLen);
 
     stream.Write(name, nameLen);
-    if (z64Required)
+    if (m_z64infoOffset > 0)
-    {
+        WriteLocalZip64ExtraInfo(stream);
-        ds.Write16(1);  // id
-        ds.Write16(16); // record size
-        ds.Write64(static_cast<wxInt64>(m_CompressedSize));
-        ds.Write64(static_cast<wxInt64>(m_Size));
-    }
     if (GetLocalExtraLen())
         stream.Write(m_LocalExtra->GetData(), GetLocalExtraLen());
 
@@ -1264,7 +1260,7 @@ size_t wxZipEntry::WriteCentral(wxOutputStream& stream, wxMBConv& conv) const
     }
 
     wxUint16 versionNeeded =
-        (z64Required) ? Z64_VERSION_NEEDED_TO_EXTRACT : GetVersionNeeded();
+        (z64Required || m_z64infoOffset) ? Z64_VERSION_NEEDED_TO_EXTRACT : GetVersionNeeded();
 
     wxDataOutputStream ds(stream);
 
@@ -1288,10 +1284,10 @@ size_t wxZipEntry::WriteCentral(wxOutputStream& stream, wxMBConv& conv) const
     {
         ds.Write16(1); // tag
         ds.Write16(z64InfoLen); // record size
-        if (m_CompressedSize > 0xffffffff)
-            ds.Write64(static_cast<wxInt64>(m_CompressedSize));
         if (m_Size > 0xffffffff)
             ds.Write64(static_cast<wxInt64>(m_Size));
+        if (m_CompressedSize > 0xffffffff)
+            ds.Write64(static_cast<wxInt64>(m_CompressedSize));
         if (m_Offset > 0xffffffff)
             ds.Write64(static_cast<wxInt64>(m_Offset));
     }
@@ -1357,12 +1353,40 @@ size_t wxZipEntry::WriteDescriptor(wxOutputStream& stream, wxUint32 crc,
     wxDataOutputStream ds(stream);
 
     ds.Write32(crc);
-    ds.Write32(wx_truncate_cast(wxUint32, compressedSize));
+    WriteLocalFileSizes(ds);
-    ds.Write32(wx_truncate_cast(wxUint32, size));
 
     return SUMS_SIZE;
 }
 
+void wxZipEntry::WriteLocalFileSizes(wxDataOutputStream& ds) const
+{
+    wxUint32 compressedSize;
+    wxUint32 size;
+    if (m_z64infoOffset > 0)
+    {
+        compressedSize = 0xffffffff;
+        size = 0xffffffff;
+    }
+    else
+    {
+        compressedSize = m_CompressedSize != wxInvalidOffset ?
+             wx_truncate_cast(wxUint32, m_CompressedSize) : 0;
+        size = m_Size != wxInvalidOffset ?
+            wx_truncate_cast(wxUint32, m_Size) : 0;
+    }
+    ds.Write32(compressedSize);
+    ds.Write32(size);
+}
+
+void  wxZipEntry::WriteLocalZip64ExtraInfo(wxOutputStream& stream) const
+{
+    wxDataOutputStream ds(stream);
+    ds.Write16(1);  // id
+    ds.Write16(16); // record size
+    ds.Write64(static_cast<wxUint64>(m_Size));
+    ds.Write64(static_cast<wxUint64>(m_CompressedSize));
+}
+
 // Returns the flags as specified or including the UTF-8 flag if filename
 // contains non ASCII characters
 wxUint16 wxZipEntry::GetInternalFlags(bool checkForUTF8) const
@@ -2173,6 +2197,7 @@ void wxZipOutputStream::Init(int level)
     m_level = level;
     m_offsetAdjustment = wxInvalidOffset;
     m_endrecWritten = false;
+    m_format = wxZIP_FORMAT_DEFAULT;
 }
 
 wxZipOutputStream::~wxZipOutputStream()
@@ -2401,7 +2426,7 @@ void wxZipOutputStream::CreatePendingEntry(const void *buffer, size_t size)
         if (spPending->m_CompressedSize != wxInvalidOffset)
             spPending->m_Flags |= wxZIP_SUMS_FOLLOW;
 
-    m_headerSize = spPending->WriteLocal(*m_parent_o_stream, GetConv());
+    m_headerSize = spPending->WriteLocal(*m_parent_o_stream, GetConv(), m_format);
     m_lasterror = m_parent_o_stream->GetLastError();
 
     if (IsOk()) {
@@ -2454,7 +2479,7 @@ void wxZipOutputStream::CreatePendingEntry()
     }
 
     spPending->m_Flags &= ~wxZIP_SUMS_FOLLOW;
-    m_headerSize = spPending->WriteLocal(*m_parent_o_stream, GetConv());
+    m_headerSize = spPending->WriteLocal(*m_parent_o_stream, GetConv(), m_format);
 
     if (m_parent_o_stream->IsOk()) {
         m_entries.push_back(spPending.release());
@@ -2552,6 +2577,12 @@ bool wxZipOutputStream::CloseEntry()
             m_parent_o_stream->SeekO(headerOffset + SUMS_OFFSET);
             entry.WriteDescriptor(*m_parent_o_stream, m_crcAccumulator,
                                   compressedSize, m_entrySize);
+            if (entry.m_z64infoOffset > 0)
+            {
+                // Skip to ZIP64 extended information extra field
+                m_parent_o_stream->SeekO(headerOffset + entry.m_z64infoOffset);
+                entry.WriteLocalZip64ExtraInfo(*m_parent_o_stream);
+            }
             m_parent_o_stream->SeekO(here);
             m_lasterror = m_parent_o_stream->GetLastError();
         } else {
@@ -2565,7 +2596,14 @@ bool wxZipOutputStream::CloseEntry()
     m_store->Close();
     m_raw = false;
 
-    if (IsOk())
+    if (IsOk() && entry.m_z64infoOffset == 0 &&
+        (entry.m_CompressedSize > 0xffffffff || entry.m_Size > 0xffffffff))
+    {
+        wxLogError(_("error writing zip entry '%s': file too large without ZIP64"),
+            entry.GetName().c_str());
+        m_lasterror = wxSTREAM_WRITE_ERROR;
+    }
+    else if (IsOk())
         m_lasterror = m_parent_o_stream->GetLastError();
     else
         wxLogError(_("error writing zip entry '%s': bad crc or length"),