Migration to dynamic translation tables continues...

ZRColaCompile/ZRColaCompile.vcxproj

@@ -60,9 +60,6 @@
     <ProjectReference Include="..\lib\libZRCola\build\libZRCola.vcxproj">
       <Project>{3c61929e-7289-4101-8d0a-da22d6e1aea8}</Project>
     </ProjectReference>
-    <ProjectReference Include="..\lib\stdex\build\stdex.vcxproj">
-      <Project>{518777cc-0a59-4415-a12a-82751ed75343}</Project>
-    </ProjectReference>
     <ProjectReference Include="..\lib\wxExtend\build\wxExtend.vcxproj">
       <Project>{a3a36689-ac35-4026-93da-a3ba0c0e767c}</Project>
     </ProjectReference>

ZRColaCompile/dbsource.cpp

@@ -32,32 +32,31 @@ ZRCola::DBSource::~DBSource()
 }
 
 
-bool ZRCola::DBSource::Open(const wxString& filename)
+bool ZRCola::DBSource::Open(LPCTSTR _filename)
 {
     wxASSERT_MSG(!m_db, wxT("database already open"));
 
-    HRESULT hr;
-
     // Create COM object.
-    hr = ::CoCreateInstance(CLSID_CADOConnection, NULL, CLSCTX_ALL, IID_IADOConnection, (LPVOID*)&m_db);
+    HRESULT hr = ::CoCreateInstance(CLSID_CADOConnection, NULL, CLSCTX_ALL, IID_IADOConnection, (LPVOID*)&m_db);
     if (SUCCEEDED(hr)) {
         // Open the database.
         std::wstring cn;
         cn  = L"Driver={Microsoft Access Driver (*.mdb)};";
         cn += L"Dbq=";
-        cn += filename.c_str();
+        cn += _filename;
         cn += L";Uid=;Pwd=;";
         hr = m_db->Open(ATL::CComBSTR(cn.c_str()));
         if (SUCCEEDED(hr)) {
             // Database open and ready.
+            filename = _filename;
             return true;
         } else {
-            wxLogMessage(wxT("Could not open database %s (0x%x)."), filename.c_str(), hr);
+            _ftprintf(stderr, wxT("%s: error ZCC0002: Could not open database (0x%x).\n"), (LPCTSTR)_filename, hr);
             LogErrors();
         }
         m_db.Release();
     } else
-        wxLogMessage(wxT("Creating ADOConnection object failed (0x%x)."), hr);
+        _ftprintf(stderr, wxT("%s: error ZCC0001: Creating ADOConnection object failed (0x%x).\n"), (LPCTSTR)_filename, hr);
 
     return false;
 }
@@ -85,7 +84,7 @@ void ZRCola::DBSource::LogErrors() const
                 ATL::CComBSTR desc;
                 wxVERIFY(SUCCEEDED(err->get_Description(&desc)));
 
-                wxLogMessage(wxT("ADO Error 0x%x: %ls"), num, (BSTR)desc);
+                _ftprintf(stderr, wxT("    error ADO%x: %ls\n"), num, (BSTR)desc);
 
                 err->Release();
             }
@@ -103,10 +102,78 @@ bool ZRCola::DBSource::SelectCompositions(ATL::CComPtr<ADORecordset> &rs) const
     wxCHECK(SUCCEEDED(::CoCreateInstance(CLSID_CADORecordset, NULL, CLSCTX_ALL, IID_IADORecordset, (LPVOID*)&rs)), false);
 
     // Open it.
-    if (FAILED(rs->Open(ATL::CComVariant(L"SELECT [komb], [znak] FROM [VRS_ReplChar] WHERE [rang_komb]=1 ORDER BY [komb] ASC"), ATL::CComVariant(m_db), adOpenForwardOnly, adLockReadOnly, adCmdText))) {
+    if (FAILED(rs->Open(ATL::CComVariant(L"SELECT [komb], [znak] FROM [VRS_ReplChar] WHERE [rang_komb]=1 ORDER BY [komb] ASC"), ATL::CComVariant(m_db), adOpenStatic, adLockReadOnly, adCmdText))) {
+        _ftprintf(stderr, wxT("%s: error ZCC0010: Error loading compositions from database. Please make sure the file is ZRCola.zrc compatible.\n"), filename.c_str());
         LogErrors();
         return false;
     }
 
     return true;
 }
+
+
+bool ZRCola::DBSource::GetComposition(const ATL::CComPtr<ADORecordset>& rs, ZRCola::DBSource::composition& comp) const
+{
+    wxASSERT_MSG(rs, wxT("recordset is empty"));
+
+    CComPtr<ADOFields> flds;
+    wxVERIFY(SUCCEEDED(rs->get_Fields(&flds)));
+
+    {
+        CComPtr<ADOField> f;
+        wxVERIFY(SUCCEEDED(flds->get_Item(CComVariant(L"komb"), &f)));
+        CComVariant v;
+        wxVERIFY(SUCCEEDED(f->get_Value(&v)));
+
+        // Parse "komb" field. Must be "xxxx+xxxx+xxxx..." sequence.
+        wxVERIFY(SUCCEEDED(v.ChangeType(VT_BSTR)));
+        comp.src.clear();
+        for (UINT i = 0, n = ::SysStringLen(V_BSTR(&v)); i < n && V_BSTR(&v)[i];) {
+            // Parse Unicode code.
+            UINT j = 0;
+            wchar_t c = 0;
+            for (; i < n && V_BSTR(&v)[i]; i++, j++) {
+                     if (L'0' <= V_BSTR(&v)[i] && V_BSTR(&v)[i] <= L'9') c = c*0x10 + (V_BSTR(&v)[i] - L'0');
+                else if (L'A' <= V_BSTR(&v)[i] && V_BSTR(&v)[i] <= L'F') c = c*0x10 + (V_BSTR(&v)[i] - L'A' + 10);
+                else if (L'a' <= V_BSTR(&v)[i] && V_BSTR(&v)[i] <= L'f') c = c*0x10 + (V_BSTR(&v)[i] - L'a' + 10);
+                else break;
+            }
+            if (j <= 0 || 4 < j) {
+                _ftprintf(stderr, wxT("%s: error ZCC0020: Syntax error in \"%ls\" field (\"%.*ls\"). Unicode code must be one to four hexadecimal characters long.\n"), filename.c_str(), L"komb", n, V_BSTR(&v));
+                return false;
+            }
+            comp.src += c;
+
+            // Skip delimiter(s) and whitespace.
+            for (; i < n && V_BSTR(&v)[i] && (V_BSTR(&v)[i] == L'+' || iswspace(V_BSTR(&v)[i])); i++);
+        }
+    }
+
+    {
+        CComPtr<ADOField> f;
+        wxVERIFY(SUCCEEDED(flds->get_Item(CComVariant(L"znak"), &f)));
+        CComVariant v;
+        wxVERIFY(SUCCEEDED(f->get_Value(&v)));
+
+        // Parse "znak" field. Must be exactly one Unicode code.
+        wxVERIFY(SUCCEEDED(v.ChangeType(VT_BSTR)));
+        UINT i = 0, n = ::SysStringLen(V_BSTR(&v));
+        wchar_t c = 0;
+        for (; i < n && V_BSTR(&v)[i]; i++) {
+                 if (L'0' <= V_BSTR(&v)[i] && V_BSTR(&v)[i] <= L'9') c = c*0x10 + (V_BSTR(&v)[i] - L'0');
+            else if (L'A' <= V_BSTR(&v)[i] && V_BSTR(&v)[i] <= L'F') c = c*0x10 + (V_BSTR(&v)[i] - L'A' + 10);
+            else if (L'a' <= V_BSTR(&v)[i] && V_BSTR(&v)[i] <= L'f') c = c*0x10 + (V_BSTR(&v)[i] - L'a' + 10);
+            else break;
+        }
+        if (i <= 0 && 4 < i) {
+            _ftprintf(stderr, wxT("%s: error ZCC0020: Syntax error in \"%ls\" field (\"%.*ls\"). Unicode code must be one to four hexadecimal characters long.\n"), filename.c_str(), L"znak", n, V_BSTR(&v));
+            return false;
+        } else if (i != n) {
+            _ftprintf(stderr, wxT("%s: error ZCC0021: Syntax error in \"%ls\" field (\"%.*ls\"). Extra trailing characters.\n"), filename.c_str(), L"znak", n, V_BSTR(&v));
+            return false;
+        }
+        comp.dst = c;
+    }
+
+    return true;
+}

ZRColaCompile/dbsource.h

@@ -19,18 +19,29 @@
 
 #pragma once
 
-#include <wx/string.h>
-
 #include <atlbase.h>
-
 #include <adoint.h>
 
+#include <algorithm>
+#include <ostream>
+
 
 namespace ZRCola {
     ///
     /// Source database
     ///
-    class DBSource {
+    class DBSource
+    {
+    public:
+        ///
+        /// Composition
+        ///
+        class composition {
+        public:
+            std::wstring src;   ///< decomposed string
+            wchar_t dst;        ///< composed character
+        };
+
     public:
         DBSource();
         virtual ~DBSource();
@@ -44,13 +55,41 @@ namespace ZRCola {
         /// - true when open succeeds
         /// - false otherwise
         ///
-        bool Open(const wxString& filename);
+        bool Open(LPCTSTR filename);
 
         ///
         /// Logs errors in database connections
         ///
         void LogErrors() const;
 
+        ///
+        /// Is recordset at end
+        ///
+        /// \param[out] rs  Recordset with results
+        ///
+        /// \returns
+        /// - true when at end
+        /// - false otherwise
+        ///
+        static inline bool IsEOF(const ATL::CComPtr<ADORecordset>& rs)
+        {
+            VARIANT_BOOL eof = VARIANT_TRUE;
+            return FAILED(rs->get_EOF(&eof)) || eof ? true : false;
+        }
+
+        ///
+        /// Gets number of records in a recordset
+        ///
+        /// \param[out] rs  Recordset with results
+        ///
+        /// \returns Number of records
+        ///
+        static inline size_t GetRecordsetCount(const ATL::CComPtr<ADORecordset>& rs)
+        {
+            ADO_LONGPTR count;
+            return SUCCEEDED(rs->get_RecordCount(&count)) ? count : (size_t)-1;
+        }
+
         ///
         /// Returns ordered decomposed to composed character translations
         ///
@@ -60,9 +99,53 @@ namespace ZRCola {
         /// - true when query succeeds
         /// - false otherwise
         ///
-        bool SelectCompositions(ATL::CComPtr<ADORecordset> &rs) const;
+        bool SelectCompositions(ATL::CComPtr<ADORecordset>& rs) const;
+
+        ///
+        /// Returns composition data
+        ///
+        /// \param[in]  rs    Recordset with results
+        /// \param[out] comp  Composition
+        ///
+        /// \returns
+        /// - true when succeeded
+        /// - false otherwise
+        ///
+        bool GetComposition(const ATL::CComPtr<ADORecordset>& rs, composition& comp) const;
 
     protected:
-        ATL::CComPtr<ADOConnection> m_db;    ///< the database
+        std::basic_string<TCHAR> filename;  ///< the database filename
+        ATL::CComPtr<ADOConnection> m_db;   ///< the database
     };
 };
+
+
+///
+/// Stores composition data
+///
+/// \param[in] stream  Output stream
+/// \param[in] comp    Composition
+///
+/// \returns The stream \p stream
+///
+inline std::ostream& operator <<(std::ostream& stream, const ZRCola::DBSource::composition& comp)
+{
+    // Store src.
+    size_t len = comp.src.length();
+    if (len > 0xffff) {
+        // Decomposed string is too long.
+        stream.setstate(std::ios::failbit);
+        return stream;
+    }
+    unsigned short count = (unsigned short)len;
+    if (stream.fail()) return stream;
+    stream.write((const char*)&count, sizeof(count));
+    if (stream.fail()) return stream;
+    stream.write((const char*)comp.src.c_str(), sizeof(wchar_t) * count);
+
+    // Store dst.
+    if (stream.fail()) return stream;
+    stream.write((const char*)&comp.dst, sizeof(comp.dst));
+
+    return stream;
+}

ZRColaCompile/main.cpp

@@ -83,18 +83,84 @@ int _tmain(int argc, _TCHAR *argv[])
         return 1;
     }
 
-    wxFile dst;
     const wxString& filenameOut = parser.GetParam(1);
-    if (!dst.Create(filenameOut, true, wxS_IRUSR | wxS_IWUSR | wxS_IRGRP | wxS_IWGRP | wxS_IROTH)) {
+    std::fstream dst((LPCTSTR)filenameOut, std::ios_base::out | std::ios_base::trunc | std::ios_base::binary);
+    if (dst.fail()) {
         _ftprintf(stderr, _("Error opening %s output file.\n"), filenameOut.fn_str());
         return 1;
     }
 
     ATL::CComPtr<ADORecordset> rs_comp;
-    if (!src.SelectCompositions(rs_comp)) {
+    wxCHECK(src.SelectCompositions(rs_comp), 1);
-        _ftprintf(stderr, _("Error loading compositions from %s input file. Please make sure the input file is ZRCola.zrc compatible.\n"), filenameIn.fn_str());
+
-        return 1;
+    bool has_errors = false;
+
+    // Open file ID.
+    std::streamoff dst_start = stdex::idrec::open<ZRCola::recordid_t, ZRCola::recordsize_t>(dst, ZRCOLA_DB_ID);
+
+    // Get number of compositions.
+    size_t comp_count = src.GetRecordsetCount(rs_comp);
+    if (comp_count < 0xffffffff) { // 4G check (-1 is reserved for error condition)
+        // Allocate memory.
+        std::vector<ZRCola::composition_index> comp_index;
+        comp_index.reserve(comp_count);
+        std::vector<wchar_t> comp_data;
+        comp_data.reserve(comp_count*4);
+        ZRCola::DBSource::composition comp;
+
+        // Parse compositions and build index and data.
+        while (!ZRCola::DBSource::IsEOF(rs_comp)) {
+            // Read composition from the database.
+            if (src.GetComposition(rs_comp, comp)) {
+                // Add composition to index and data.
+                ZRCola::composition_index ci;
+                ci.src = (unsigned int)comp_data.size();
+                for (std::wstring::size_type i = 0, n = comp.src.length(); i < n; i++)
+                    comp_data.push_back(comp.src[i]);
+                ci.dst = (unsigned int)comp_data.size();
+                comp_data.push_back(comp.dst);
+                comp_index.push_back(ci);
+            } else
+                has_errors = true;
+
+            wxVERIFY(SUCCEEDED(rs_comp->MoveNext()));
+        }
+
+        // Write compositions to file.
+        std::streamoff start = stdex::idrec::open<ZRCola::recordid_t, ZRCola::recordsize_t>(dst, ZRCOLA_DB_COMPOSITIONS_ID);
+        {
+            unsigned int _count = comp_count;
+            dst.write((const char*)&_count, sizeof(_count));
+            dst.write((const char*)comp_index.data(), sizeof(ZRCola::composition_index)*_count);
+        }
+        {
+            std::vector<wchar_t>::size_type count = comp_data.size();
+            if (count <= 0xffffffff) { // 4G check
+                unsigned int _count = (unsigned int)count;
+                dst.write((const char*)&_count, sizeof(_count));
+                dst.write((const char*)comp_data.data(), sizeof(wchar_t)*_count);
+            } else {
+                _ftprintf(stderr, wxT("%s: error ZCC0004: Composition data exceeds 4G. Please make sure the file is ZRCola.zrc compatible.\n"), (LPCTSTR)filenameIn.c_str());
+                has_errors = true;
+            }
+        }
+        stdex::idrec::close<ZRCola::recordid_t, ZRCola::recordsize_t, ZRCOLA_RECORD_ALIGN>(dst, start);
+    } else {
+        _ftprintf(stderr, wxT("%s: error ZCC0003: Error getting composition count from database or too many compositions. Please make sure the file is ZRCola.zrc compatible.\n"), (LPCTSTR)filenameIn.c_str());
+        has_errors = true;
     }
 
-    return 0;
+    stdex::idrec::close<ZRCola::recordid_t, ZRCola::recordsize_t, ZRCOLA_RECORD_ALIGN>(dst, dst_start);
+
+    if (dst.fail()) {
+        _ftprintf(stderr, wxT("Writing to %s output file failed.\n"), (LPCTSTR)filenameOut.c_str());
+        has_errors = true;
+    }
+
+    if (has_errors) {
+        dst.close();
+        wxRemoveFile(filenameOut);
+        return 1;
+    } else
+        return 0;
 }

ZRColaCompile/stdafx.h

@@ -22,16 +22,21 @@
 #include "../include/zrcola.h"
 #include "dbsource.h"
 
+#include <zrcola/compose.h>
+
 #include <wx/app.h>
 #include <wx/cmdline.h>
-#include <wx/file.h>
 #include <wx/intl.h>
 
 #include <wxex/common.h>
 #include <wxex/comutils.h>
 
+#include <stdex/idrec.h>
+
 #include <initguid.h> // GUID helper to prevent LNK2001 errors (unresolved external symbol IID_IADO...)
 #include <adoint.h>
 #include <adoid.h>
 
 #include <tchar.h>
+
+#include <fstream>

lib/libZRCola/build/libZRCola.props

@@ -10,6 +10,7 @@
   <ItemDefinitionGroup>
     <ClCompile>
       <PreprocessorDefinitions>LIBZRCOLA;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>..\..\stdex\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
   </ItemDefinitionGroup>
   <ItemGroup>

lib/libZRCola/include/zrcola/common.h

@@ -35,7 +35,22 @@
 #pragma warning(disable: 4251)
 
 
+///
+/// Data records alignment
+///
+#define ZRCOLA_RECORD_ALIGN 1
+
+///
+/// Database IDs
+///
+#define ZRCOLA_DB_ID                ((ZRCola::recordid_t)0x0043525a)
+#define ZRCOLA_DB_COMPOSITIONS_ID   ((ZRCola::recordid_t)0x00000001)
+
+
 namespace ZRCola {
+    typedef unsigned __int32 recordid_t;
+    typedef unsigned __int32 recordsize_t;
+
     ///
     /// Composed-decomposed index transformation mapping
     ///

lib/libZRCola/include/zrcola/compose.h

@@ -26,6 +26,29 @@
 
 
 namespace ZRCola {
+    ///
+    /// Composition
+    ///
+    struct composition {
+        const wchar_t *src; ///< Decomposed string
+        wchar_t dst;        ///< Composed character
+    };
+
+
+#pragma pack(push)
+#pragma pack(4)
+
+    ///
+    /// Composition index
+    ///
+    struct composition_index {
+        unsigned int src;   ///< Decomposed string offset
+        unsigned int dst;   ///< Composed character offset
+    };
+
+#pragma pack(pop)
+
+
     ///
     /// Composes string
     ///

lib/libZRCola/include/zrcola/decompose.h

@@ -26,6 +26,15 @@
 
 
 namespace ZRCola {
+    ///
+    /// Decomposition
+    ///
+    struct decomposition {
+        wchar_t src;        ///< composed character
+        const wchar_t *dst; ///< decomposed string
+    };
+
+
     ///
     /// Decomposes string
     ///

lib/libZRCola/src/stdafx.h

@@ -26,19 +26,9 @@
 
 
 namespace ZRCola {
-    struct composition {
-        const wchar_t *src;
-        wchar_t dst;
-    };
-
     extern const composition* compositions;
     extern const size_t compositionsCount;
 
-    struct decomposition {
-        wchar_t src;
-        const wchar_t *dst;
-    };
-
     extern const decomposition* decompositions;
     extern const size_t decompositionsCount;
 }

lib/stdex

@@ -1 +1 @@
-Subproject commit f86db2052780c33fae23815f066c7b71002acdd8
+Subproject commit 72766c21b2889ac13c94f4645638fe043dfa35d7