Introduce bad ZRCola Unicode Composition highlighting

Signed-off-by: Simon Rozman <simon@rozman.si>

ZRCola/locale/ZRCola.pot

@@ -2,7 +2,7 @@
 msgid ""
 msgstr ""
 "Project-Id-Version: ZRCola\n"
-"POT-Creation-Date: 2021-12-20 20:17+0100\n"
+"POT-Creation-Date: 2021-12-21 14:16+0100\n"
 "PO-Revision-Date: 2019-04-01 19:38+0200\n"
 "Last-Translator: Simon Rozman <simon.rozman@amebis.si>\n"
 "Language-Team: Amebis, d. o. o., Kamnik <info@amebis.si>\n"
@@ -108,36 +108,37 @@ msgstr "Prekini sestavljanje in vrni fokus nazaj izvornemu oknu"
 #: res/zrcolagui.cpp:138 zrcolagui.cpp:138
 #, fuzzy
 msgid "(De)&composition"
-msgstr "Vklopi/izklopi (raz-)sestavljanje ZRCola"
+msgstr "(&Raz-)sestavljanje"
 
 #: res/zrcolagui.cpp:139 zrcolagui.cpp:139
 #, fuzzy
 msgid "&None"
-msgstr "(brez)"
+msgstr "&Brez"
 
 #: res/zrcolagui.cpp:139 zrcolagui.cpp:139
 #, fuzzy
 msgid "No character (De)composition"
-msgstr "Vklopi/izklopi (raz-)sestavljanje ZRCola"
+msgstr "Brez (raz-)sestavljanja znakov"
 
 #: res/zrcolagui.cpp:142 zrcolagui.cpp:142
 #, fuzzy
 msgid "&ZRCola"
-msgstr "ZRCola"
+msgstr "&ZRCola"
 
 #: res/zrcolagui.cpp:142 zrcolagui.cpp:142
 #, fuzzy
 msgid "ZRCola character (De)composition"
-msgstr "Vklopi/izklopi (raz-)sestavljanje ZRCola"
+msgstr "(Raz-)sestavljanje znakov ZRCola"
 
 #: res/zrcolagui.cpp:145 zrcolagui.cpp:145
 #, fuzzy
 msgid "&Unicode"
-msgstr "Unicode"
+msgstr "&Unicode"
 
 #: res/zrcolagui.cpp:145 zrcolagui.cpp:145
+#, fuzzy
 msgid "Unicode character (De)composition"
-msgstr ""
+msgstr "(Raz-)sestavljanje znakov Unicode"
 
 #: res/zrcolagui.cpp:151 zrcolagui.cpp:151
 #, fuzzy
@@ -308,7 +309,7 @@ msgstr "Pošlji razstavljeno"
 #: res/zrcolagui.cpp:232 zrcolagui.cpp:232
 #, fuzzy
 msgid "No (De)composition"
-msgstr "Prekini raz/sestavljanje"
+msgstr "Brez (raz-)sestavljanja"
 
 #: res/zrcolagui.cpp:232 res/zrcolagui.cpp:940 res/zrcolagui.h:119
 #: zrcolaapp.cpp:60 zrcolafrm.cpp:118 zrcolagui.cpp:232 zrcolagui.cpp:943
@@ -807,12 +808,12 @@ msgstr "Opozorilo"
 msgid "ZRCola keyboard shortcut Win+F6 could not be registered. Some functionality will not be available."
 msgstr "ZRColine bližnjice na tipkovnici Win+F6 ni mogoče registrirati. Nekaj funkcionalnosti ne bo na voljo."
 
-#: zrcolafrm.cpp:508
+#: zrcolafrm.cpp:513
 #, fuzzy
 msgid "http://zrcola.zrc-sazu.si/en/info/instructions/"
 msgstr "http://zrcola.zrc-sazu.si/info/instructions/"
 
-#: zrcolafrm.cpp:533
+#: zrcolafrm.cpp:538
 #, fuzzy
 msgid "http://zrcola.zrc-sazu.si/wp-content/uploads/2016/06/ZRCola_tipkovnica_Jun2016.pdf"
 msgstr "http://zrcola.zrc-sazu.si/wp-content/uploads/2016/06/ZRCola_tipkovnica_Jun2016.pdf"

ZRCola/locale/de_DE.po

@@ -4,8 +4,8 @@
 msgid ""
 msgstr ""
 "Project-Id-Version: ZRCola\n"
-"POT-Creation-Date: 2021-12-20 20:18+0100\n"
+"POT-Creation-Date: 2021-12-21 14:17+0100\n"
-"PO-Revision-Date: 2021-12-20 20:18+0100\n"
+"PO-Revision-Date: 2021-12-21 14:17+0100\n"
 "Last-Translator: Simon Rozman <simon.rozman@amebis.si>\n"
 "Language-Team: German (Germany) (https://www.transifex.com/amebis/teams/91592/de_DE/)\n"
 "Language: de_DE\n"
@@ -655,11 +655,11 @@ msgstr ""
 msgid "ZRCola keyboard shortcut Win+F6 could not be registered. Some functionality will not be available."
 msgstr ""
 
-#: zrcolafrm.cpp:508
+#: zrcolafrm.cpp:513
 msgid "http://zrcola.zrc-sazu.si/en/info/instructions/"
 msgstr "http://zrcola.zrc-sazu.si/de/info/instructions/"
 
-#: zrcolafrm.cpp:533
+#: zrcolafrm.cpp:538
 msgid "http://zrcola.zrc-sazu.si/wp-content/uploads/2016/06/ZRCola_tipkovnica_Jun2016.pdf"
 msgstr "http://zrcola.zrc-sazu.si/wp-content/uploads/2016/06/ZRCola_tipkovnica_Jun2016.pdf"
 

ZRCola/locale/ru_RU.po

@@ -5,8 +5,8 @@
 msgid ""
 msgstr ""
 "Project-Id-Version: ZRCola\n"
-"POT-Creation-Date: 2021-12-20 20:18+0100\n"
+"POT-Creation-Date: 2021-12-21 14:17+0100\n"
-"PO-Revision-Date: 2021-12-20 20:18+0100\n"
+"PO-Revision-Date: 2021-12-21 14:17+0100\n"
 "Last-Translator: Simon Rozman <simon.rozman@amebis.si>\n"
 "Language-Team: Russian (Russia) (https://www.transifex.com/amebis/teams/91592/ru_RU/)\n"
 "Language: ru_RU\n"
@@ -687,11 +687,11 @@ msgstr "Предупреждение"
 msgid "ZRCola keyboard shortcut Win+F6 could not be registered. Some functionality will not be available."
 msgstr "Сочетание клавиш Win+F6 невозможно регистрировать. Некоторые функциональности не будут доступны."
 
-#: zrcolafrm.cpp:508
+#: zrcolafrm.cpp:513
 msgid "http://zrcola.zrc-sazu.si/en/info/instructions/"
 msgstr "http://zrcola.zrc-sazu.si/ru/info/instructions/"
 
-#: zrcolafrm.cpp:533
+#: zrcolafrm.cpp:538
 msgid "http://zrcola.zrc-sazu.si/wp-content/uploads/2016/06/ZRCola_tipkovnica_Jun2016.pdf"
 msgstr "http://zrcola.zrc-sazu.si/wp-content/uploads/2016/06/ZRCola_tipkovnica_Jun2016.pdf"
 

ZRCola/locale/sl_SI.po

@@ -4,8 +4,8 @@
 msgid ""
 msgstr ""
 "Project-Id-Version: ZRCola\n"
-"POT-Creation-Date: 2021-12-20 20:18+0100\n"
+"POT-Creation-Date: 2021-12-21 14:17+0100\n"
-"PO-Revision-Date: 2021-12-20 20:20+0100\n"
+"PO-Revision-Date: 2021-12-21 14:17+0100\n"
 "Last-Translator: Simon Rozman <simon.rozman@amebis.si>\n"
 "Language-Team: Slovenian (Slovenia) (https://www.transifex.com/amebis/teams/91592/sl_SI/)\n"
 "Language: sl_SI\n"
@@ -666,11 +666,11 @@ msgstr "Opozorilo"
 msgid "ZRCola keyboard shortcut Win+F6 could not be registered. Some functionality will not be available."
 msgstr "ZRColine bližnjice na tipkovnici Win+F6 ni mogoče registrirati. Nekaj funkcionalnosti ne bo na voljo."
 
-#: zrcolafrm.cpp:508
+#: zrcolafrm.cpp:513
 msgid "http://zrcola.zrc-sazu.si/en/info/instructions/"
 msgstr "http://zrcola.zrc-sazu.si/info/instructions/"
 
-#: zrcolafrm.cpp:533
+#: zrcolafrm.cpp:538
 msgid "http://zrcola.zrc-sazu.si/wp-content/uploads/2016/06/ZRCola_tipkovnica_Jun2016.pdf"
 msgstr "http://zrcola.zrc-sazu.si/wp-content/uploads/2016/06/ZRCola_tipkovnica_Jun2016.pdf"
 

ZRCola/zrcolaapp.cpp

@@ -144,6 +144,12 @@ bool ZRColaApp::OnInit()
                             wxFAIL_MSG(wxT("Error reading tag name data from ZRCola.zrcdb."));
                             m_tn_db.clear();
                         }
+                    } else if (id == ZRCola::highlight_rec::id) {
+                        dat >> ZRCola::highlight_rec(m_h_db);
+                        if (!dat.good()) {
+                            wxFAIL_MSG(wxT("Error reading highlight data from ZRCola.zrcdb."));
+                            m_h_db.clear();
+                        }
                     } else
                         stdex::idrec::ignore<ZRCola::recordsize_t, ZRCOLA_RECORD_ALIGN>(dat);
                 }

ZRCola/zrcolaapp.h

@@ -21,6 +21,7 @@ class ZRColaApp;
 #include <wx/intl.h>
 #pragma warning(pop)
 #include <zrcola/character.h>
+#include <zrcola/highlight.h>
 #include <zrcola/language.h>
 #include <zrcola/translate.h>
 #include <zrcola/tag.h>
@@ -72,6 +73,7 @@ public:
     ZRCola::chrcat_db m_cc_db;      ///< Characted category database
     ZRCola::chrtag_db m_ct_db;      ///< Character tag database
     ZRCola::tagname_db m_tn_db;     ///< Tag name database
+    ZRCola::highlight_db m_h_db;    ///< Highlight database
 
     wxZRColaFrame *m_mainWnd;       ///< Main window
 

ZRCola/zrcolacomppnl.cpp

@@ -17,6 +17,7 @@ wxZRColaComposerPanel::wxZRColaComposerPanel(wxWindow* parent) :
     m_destinationRestyled(false),
     m_styleNormal(*wxBLACK, *wxWHITE, wxFont(20, wxFONTFAMILY_DEFAULT, wxFONTSTYLE_NORMAL, wxFONTWEIGHT_NORMAL, false, wxT("ZRCola"))),
     m_stylePUA(*wxBLUE, *wxWHITE, wxFont(20, wxFONTFAMILY_DEFAULT, wxFONTSTYLE_NORMAL, wxFONTWEIGHT_NORMAL, false, wxT("ZRCola"))),
+    m_styleZRColaUnicodeComposedIssues(*wxRED, *wxWHITE, wxFont(20, wxFONTFAMILY_DEFAULT, wxFONTSTYLE_NORMAL, wxFONTWEIGHT_NORMAL, false, wxT("ZRCola"))),
     m_selSource(0, 0),
     m_selDestination(0, 0),
     wxZRColaComposerPanelBase(parent)
@@ -320,17 +321,25 @@ void wxZRColaComposerPanel::OnDestinationText(wxCommandEvent& event)
 
     auto app = dynamic_cast<ZRColaApp*>(wxTheApp);
     m_destinationRestyled = true;
-    if (app->m_mainWnd->m_warnPUA) {
+    wxString src = m_destination->GetValue();
-        wxString src = m_destination->GetValue();
+    app->m_h_db.Highlight(src, src.Length(), [this, app, src](ZRCola::hlghtsetid_t set, size_t start, size_t end) {
-        size_t len = src.Length();
+        switch (set) {
-        for (size_t i = 0, j; i < len;) {
+        case ZRCOLA_HLGHTSETID_ZRCOLA_UNICODE_COMPOSED_ISSUES:
-            bool pua_i = ZRCola::ispua(src[i]);
+            m_destination->SetStyle((long)start, (long)end, m_styleZRColaUnicodeComposedIssues);
-            for (j = i + 1; j < len && pua_i == ZRCola::ispua(src[j]); j++);
+            break;
-            m_destination->SetStyle((long)i, (long)j, pua_i ? m_stylePUA : m_styleNormal);
+
-            i = j;
+        default:
+            if (app->m_mainWnd->m_warnPUA) {
+                for (size_t i = start, j; i < end;) {
+                    bool pua_i = ZRCola::ispua(src[i]);
+                    for (j = i + 1; j < end && pua_i == ZRCola::ispua(src[j]); j++);
+                    m_destination->SetStyle((long)i, (long)j, pua_i ? m_stylePUA : m_styleNormal);
+                    i = j;
+                }
+            } else
+                m_destination->SetStyle((long)start, (long)end, m_styleNormal);
         }
-    } else
+    });
-        m_destination->SetStyle(0, GetWindowTextLength(m_destination->GetHWND()), m_styleNormal);
     m_destinationRestyled = false;
 }
 

ZRCola/zrcolacomppnl.h

@@ -63,7 +63,8 @@ protected:
         m_destinationRestyled;                      ///< Boolean flag to mark destination text is being restyled
     wxTextAttr
         m_styleNormal,                              ///< Normal text style
-        m_stylePUA;                                 ///< PUA character text style
+        m_stylePUA,                                 ///< PUA character text style
+        m_styleZRColaUnicodeComposedIssues;         ///< ZRCola Unicode Composed issues character text style
     std::vector<ZRCola::mapping_vector> m_mapping;  ///< Character index mapping vector between source and normalized text
     std::pair<long, long>
         m_selSource,                                ///< Character index of selected text in source text control

ZRColaCompile/dbsource.cpp

@@ -242,6 +242,8 @@ ZRCola::DBSource::~DBSource()
     m_comTranslation.free();
     m_pCharacterGroup1.free();
     m_comCharacterGroup.free();
+    m_pHighlight1.free();
+    m_comHighlight.free();
 
     if (m_db)
         m_db->Close();
@@ -326,6 +328,23 @@ bool ZRCola::DBSource::Open(LPCTSTR filename)
                 wxVERIFY(SUCCEEDED(params->Append(m_pTranslationSets1)));
             }
 
+            wxASSERT_MSG(!m_comHighlight, wxT("ADO command already created"));
+            wxVERIFY(SUCCEEDED(::CoCreateInstance(CLSID_CADOCommand, NULL, CLSCTX_ALL, IID_IADOCommand, (LPVOID*)&m_comHighlight)));
+            wxVERIFY(SUCCEEDED(m_comHighlight->put_ActiveConnection(variant(m_db))));
+            wxVERIFY(SUCCEEDED(m_comHighlight->put_CommandType(adCmdText)));
+            wxVERIFY(SUCCEEDED(m_comHighlight->put_CommandText(bstr(L"SELECT [komb] "
+                L"FROM [VRS_HighlightChars2] "
+                L"WHERE [group]=? "
+                L"ORDER BY [komb]"))));
+            {
+                // Create and add command parameters.
+                com_obj<ADOParameters> params;
+                wxVERIFY(SUCCEEDED(m_comHighlight->get_Parameters(&params)));
+                wxASSERT_MSG(!m_pHighlight1, wxT("ADO command parameter already created"));
+                wxVERIFY(SUCCEEDED(m_comHighlight->CreateParameter(bstr(L"@group"), adSmallInt, adParamInput, 0, variant(DISP_E_PARAMNOTFOUND, VT_ERROR), &m_pHighlight1)));
+                wxVERIFY(SUCCEEDED(params->Append(m_pHighlight1)));
+            }
+
             return true;
         } else {
             _ftprintf(stderr, wxT("%s: error ZCC0011: Could not open database (0x%x).\n"), (LPCTSTR)filename, hr);
@@ -1439,3 +1458,41 @@ bool ZRCola::DBSource::GetTagName(const winstd::com_obj<ADORecordset>& rs, tagna
 
     return true;
 }
+
+
+bool ZRCola::DBSource::SelectHighlights(short set, winstd::com_obj<ADORecordset>& rs) const
+{
+    // Create a new recordset.
+    rs.free();
+    wxVERIFY(SUCCEEDED(::CoCreateInstance(CLSID_CADORecordset, NULL, CLSCTX_ALL, IID_IADORecordset, (LPVOID*)&rs)));
+    wxVERIFY(SUCCEEDED(rs->put_CursorLocation(adUseClient)));
+    wxVERIFY(SUCCEEDED(rs->put_CursorType(adOpenForwardOnly)));
+    wxVERIFY(SUCCEEDED(rs->put_LockType(adLockReadOnly)));
+
+    // Open it.
+    wxVERIFY(SUCCEEDED(m_pHighlight1->put_Value(variant(set))));
+    if (FAILED(rs->Open(variant(m_comHighlight), variant(DISP_E_PARAMNOTFOUND, VT_ERROR)))) {
+        _ftprintf(stderr, wxT("%s: error ZCC0101: Error loading highlights from database. Please make sure the file is ZRCola.zrc compatible.\n"), m_filename.c_str());
+        LogErrors();
+        return false;
+    }
+
+    return true;
+}
+
+
+bool ZRCola::DBSource::GetHighlight(const com_obj<ADORecordset>& rs, ZRCola::DBSource::highlight& h) const
+{
+    wxASSERT_MSG(rs, wxT("recordset is empty"));
+
+    com_obj<ADOFields> flds;
+    wxVERIFY(SUCCEEDED(rs->get_Fields(&flds)));
+
+    {
+        com_obj<ADOField> f;
+        wxVERIFY(SUCCEEDED(flds->get_Item(variant(L"komb"), &f)));
+        wxCHECK(GetUnicodeString(f, h.chr), false);
+    }
+
+    return true;
+}

ZRColaCompile/dbsource.h

@@ -6,7 +6,7 @@
 #pragma once
 
 #include <zrcola/character.h>
-#include <zrcola/common.h>
+#include <zrcola/highlight.h>
 #include <zrcola/language.h>
 #include <zrcola/tag.h>
 #include <zrcola/translate.h>
@@ -404,6 +404,18 @@ namespace ZRCola {
         };
 
 
+        ///
+        /// Highlight
+        ///
+        class highlight {
+        public:
+            short set;          ///< Highlight set ID
+            std::wstring chr;   ///< Character sequence
+
+            inline highlight() : set((short)ZRCOLA_HLGHTSETID_DEFAULT) {}
+        };
+
+
     public:
         DBSource();
         virtual ~DBSource();
@@ -452,18 +464,6 @@ namespace ZRCola {
             return SUCCEEDED(rs->get_RecordCount(&count)) ? count : (size_t)-1;
         }
 
-        ///
-        /// Splits string to individual keywords
-        ///
-        /// \param[in ] str       String
-        /// \param[out] keywords  Array of keywords
-        ///
-        /// \returns
-        /// - true when successful
-        /// - false otherwise
-        ///
-        static bool GetKeywords(const wchar_t *str, std::vector< std::wstring > &keywords);
-
         ///
         /// Gets boolean from ZRCola.zrc database
         ///
@@ -872,6 +872,30 @@ namespace ZRCola {
         ///
         bool GetTagName(const winstd::com_obj<ADORecordset>& rs, tagname& tn) const;
 
+        ///
+        /// Returns character highlights by set
+        ///
+        /// \param[in ] set  Highlight set ID
+        /// \param[out] rs   Recordset with results
+        ///
+        /// \returns
+        /// - true when query succeeds
+        /// - false otherwise
+        ///
+        bool SelectHighlights(short set, winstd::com_obj<ADORecordset>& rs) const;
+
+        ///
+        /// Returns highlight data
+        ///
+        /// \param[in]  rs  Recordset with results
+        /// \param[out] h   Highlight
+        ///
+        /// \returns
+        /// - true when succeeded
+        /// - false otherwise
+        ///
+        bool GetHighlight(const winstd::com_obj<ADORecordset>& rs, highlight& h) const;
+
     protected:
         std::basic_string<TCHAR> m_filename;    ///< Database filename
         winstd::com_obj<ADOConnection> m_db;    ///< Database
@@ -886,6 +910,9 @@ namespace ZRCola {
         winstd::com_obj<ADOCommand> m_comTranslationSets;   ///< ADO Command for GetTranslationSeq subquery
         winstd::com_obj<ADOParameter> m_pTranslationSets1;  ///< \c m_comTranslationSets parameter
 
+        winstd::com_obj<ADOCommand> m_comHighlight;         ///< ADO Command for SelectHighlights subquery
+        winstd::com_obj<ADOParameter> m_pHighlight1;        ///< \c m_comHighlights parameter
+
         std::set<std::wstring> m_terms_ignore;  ///< Terms to ignore when comparing characters
     };
 };
@@ -1098,3 +1125,17 @@ inline ZRCola::tagname_db& operator<<(_Inout_ ZRCola::tagname_db &db, _In_ const
 
     return db;
 }
+
+
+inline ZRCola::highlight_db& operator<<(_Inout_ ZRCola::highlight_db &db, _In_ const ZRCola::DBSource::highlight &rec)
+{
+    unsigned __int32 idx = db.data.size();
+    db.data.push_back((unsigned __int16)rec.set);
+    std::wstring::size_type n = rec.chr.length();
+    wxASSERT_MSG(n <= 0xffff, wxT("character overflow"));
+    db.data.push_back((unsigned __int16)n);
+    db.data.insert(db.data.end(), rec.chr.cbegin(), rec.chr.cend());
+    db.idxChr.push_back(idx);
+
+    return db;
+}

ZRColaCompile/main.cpp

@@ -374,9 +374,9 @@ int _tmain(int argc, _TCHAR *argv[])
                 }
 
                 // Preallocate memory.
-                db_trans.idxSrc.reserve(count);
+                db_trans.idxSrc.reserve(count*2);
-                db_trans.idxDst.reserve(count);
+                db_trans.idxDst.reserve(count*2);
-                db_trans.data  .reserve(count*5);
+                db_trans.data  .reserve(count*2*8);
 
                 // Parse translations and build index and data.
                 ZRCola::DBSource::translation trans;
@@ -399,6 +399,7 @@ int _tmain(int argc, _TCHAR *argv[])
                             }
                         if (!has_pua) {
                             trans.set = (short)ZRCOLA_TRANSETID_UNICODE;
+                            trans.dst.rank += 50;
                             db_trans << trans;
                         }
                     }
@@ -416,7 +417,13 @@ int _tmain(int argc, _TCHAR *argv[])
     {
         com_obj<ADORecordset> rs_tran;
         if (src.SelectTranslations(static_cast<short>(ZRCOLA_TRANSETID_UNICODE), rs_tran)) {
-            if (src.GetRecordsetCount(rs_tran) < 0xffffffff) { // 4G check (-1 is reserved for error condition)
+            size_t count = src.GetRecordsetCount(rs_tran);
+            if (count < 0xffffffff) { // 4G check (-1 is reserved for error condition)
+                // Preallocate memory.
+                db_trans.idxSrc.reserve(db_trans.idxSrc.size() + count);
+                db_trans.idxDst.reserve(db_trans.idxDst.size() + count);
+                db_trans.data  .reserve(db_trans.data.size()   + count*8);
+
                 // Parse translations and build temporary database.
                 ZRCola::DBSource::translation trans;
                 trans.set = (short)ZRCOLA_TRANSETID_UNICODE;
@@ -467,7 +474,13 @@ int _tmain(int argc, _TCHAR *argv[])
                         // Get translations.
                         com_obj<ADORecordset> rs_tran;
                         if (src.SelectTranslations(ts.set, rs_tran)) {
-                            if (src.GetRecordsetCount(rs_tran) < 0xffffffff) { // 4G check (-1 is reserved for error condition)
+                            count = src.GetRecordsetCount(rs_tran);
+                            if (count < 0xffffffff) { // 4G check (-1 is reserved for error condition)
+                                // Preallocate memory.
+                                db_trans.idxSrc.reserve(db_trans.idxSrc.size() + count);
+                                db_trans.idxDst.reserve(db_trans.idxDst.size() + count);
+                                db_trans.data  .reserve(db_trans.data.size()   + count*8);
+
                                 // Parse translations and build temporary database.
                                 ZRCola::DBSource::translation trans;
                                 trans.set = ts.set;
@@ -500,17 +513,13 @@ int _tmain(int argc, _TCHAR *argv[])
         }
     }
 
-    // Sort indices.
-    db_transset.idxTranSet.sort();
-
     // Write translation sets to file.
+    db_transset.idxTranSet.sort();
     dst << ZRCola::transet_rec(db_transset);
 
-    // Sort indices.
+    // Write translations to file.
     db_trans.idxSrc.sort();
     db_trans.idxDst.sort();
-
-    // Write translations to file.
     dst << ZRCola::translation_rec(db_trans);
 
     {
@@ -540,11 +549,9 @@ int _tmain(int argc, _TCHAR *argv[])
                         has_errors = true;
                 }
 
-                // Sort indices.
+                // Write translation sequences to file.
                 db.idxTranSeq.sort();
                 db.idxRank   .sort();
-
-                // Write translation sequences to file.
                 dst << ZRCola::transeq_rec(db);
             } else {
                 _ftprintf(stderr, wxT("%s: error ZCC0025: Error getting translation sequence count from database or too many translation sequences.\n"), (LPCTSTR)filenameIn.c_str());
@@ -639,10 +646,8 @@ int _tmain(int argc, _TCHAR *argv[])
                         has_errors = true;
                 }
 
-                // Sort indices.
-                db.idxLang.sort();
-
                 // Write languages to file.
+                db.idxLang.sort();
                 dst << ZRCola::language_rec(db);
             } else {
                 _ftprintf(stderr, wxT("%s: error ZCC0009: Error getting language count from database or too many languages.\n"), (LPCTSTR)filenameIn.c_str());
@@ -680,13 +685,11 @@ int _tmain(int argc, _TCHAR *argv[])
                         has_errors = true;
                 }
 
-                // Sort indices.
+                // Write language characters to file.
                 db.idxChr .sort();
 #ifdef ZRCOLA_LANGCHAR_LANG_IDX
                 db.idxLang.sort();
 #endif
-
-                // Write language characters to file.
                 dst << ZRCola::langchar_rec(db);
             } else {
                 _ftprintf(stderr, wxT("%s: error ZCC0011: Error getting language characters count from database or too many langchars.\n"), (LPCTSTR)filenameIn.c_str());
@@ -729,10 +732,8 @@ int _tmain(int argc, _TCHAR *argv[])
                         has_errors = true;
                 }
 
-                // Sort indices.
-                db.idxRank.sort();
-
                 // Write character groups to file.
+                db.idxRank.sort();
                 dst << ZRCola::chrgrp_rec(db);
             } else {
                 _ftprintf(stderr, wxT("%s: error ZCC0015: Error getting character group count from database or too many character groups.\n"), (LPCTSTR)filenameIn.c_str());
@@ -753,7 +754,6 @@ int _tmain(int argc, _TCHAR *argv[])
             size_t count = src.GetRecordsetCount(rs);
             if (count < 0xffffffff) { // 4G check (-1 is reserved for error condition)
                 ZRCola::DBSource::character_desc_idx idxChrDsc, idxChrDscSub;
-
                 ZRCola::DBSource::character_bank chrs;
 
                 // Phase 1: Parse characters and build indexes.
@@ -788,14 +788,10 @@ int _tmain(int argc, _TCHAR *argv[])
                     categories_used.insert(chr->second.cat);
                 }
 
-                // Sort indices.
+                // Write characters to file.
                 db.idxChr.sort();
-
-                // Save text indices.
                 idxChrDsc   .save(db.idxDsc   );
                 idxChrDscSub.save(db.idxDscSub);
-
-                // Write characters to file.
                 dst << ZRCola::character_rec(db);
             } else {
                 _ftprintf(stderr, wxT("%s: error ZCC0017: Error getting character count from database or too many characters.\n"), (LPCTSTR)filenameIn.c_str());
@@ -839,11 +835,9 @@ int _tmain(int argc, _TCHAR *argv[])
                         has_errors = true;
                 }
 
-                // Sort indices.
+                // Write character categories to file.
                 db.idxChrCat.sort();
                 db.idxRank  .sort();
-
-                // Write character categories to file.
                 dst << ZRCola::chrcat_rec(db);
             } else {
                 _ftprintf(stderr, wxT("%s: error ZCC0019: Error getting character category count from database or too many character categories.\n"), (LPCTSTR)filenameIn.c_str());
@@ -879,11 +873,9 @@ int _tmain(int argc, _TCHAR *argv[])
                         has_errors = true;
                 }
 
-                // Sort indices.
+                // Write characters tags to file.
                 db.idxChr.sort();
                 db.idxTag.sort();
-
-                // Write characters tags to file.
                 dst << ZRCola::chrtag_rec(db);
             } else {
                 _ftprintf(stderr, wxT("%s: error ZCC0021: Error getting characters tags count from database or too many character tags.\n"), (LPCTSTR)filenameIn.c_str());
@@ -919,11 +911,9 @@ int _tmain(int argc, _TCHAR *argv[])
                         has_errors = true;
                 }
 
-                // Sort indices.
+                // Write tags to file.
                 db.idxName.sort();
                 db.idxTag .sort();
-
-                // Write tags to file.
                 dst << ZRCola::tagname_rec(db);
             } else {
                 _ftprintf(stderr, wxT("%s: error ZCC0023: Error getting tag name count from database or too many tags.\n"), (LPCTSTR)filenameIn.c_str());
@@ -935,6 +925,43 @@ int _tmain(int argc, _TCHAR *argv[])
         }
     }
 
+    {
+        // Get highlights.
+        com_obj<ADORecordset> rs;
+        if (src.SelectHighlights((short)ZRCOLA_HLGHTSETID_ZRCOLA_UNICODE_COMPOSED_ISSUES, rs)) {
+            size_t count = src.GetRecordsetCount(rs);
+            if (count < 0xffffffff) { // 4G check (-1 is reserved for error condition)
+                ZRCola::DBSource::highlight h;
+                ZRCola::highlight_db db;
+
+                // Preallocate memory.
+                db.idxChr.reserve(count);
+                db.data  .reserve(count*5);
+
+                // Parse highlights and build index and data.
+                h.set = (short)ZRCOLA_HLGHTSETID_ZRCOLA_UNICODE_COMPOSED_ISSUES;
+                for (; !ZRCola::DBSource::IsEOF(rs); rs->MoveNext()) {
+                    // Read tag name from the database.
+                    if (src.GetHighlight(rs, h)) {
+                        // Add highlight to index and data.
+                        db << h;
+                    } else
+                        has_errors = true;
+                }
+
+                // Write highlights to file.
+                db.idxChr.sort();
+                dst << ZRCola::highlight_rec(db);
+            } else {
+                _ftprintf(stderr, wxT("%s: error ZCC0027: Error getting highlight count from database or too many tags.\n"), (LPCTSTR)filenameIn.c_str());
+                has_errors = true;
+            }
+        } else {
+            _ftprintf(stderr, wxT("%s: error ZCC0026: Error getting highlights from database. Please make sure the file is ZRCola.zrc compatible.\n"), (LPCTSTR)filenameIn.c_str());
+            has_errors = true;
+        }
+    }
+
     idrec::close<ZRCola::recordid_t, ZRCola::recordsize_t, ZRCOLA_RECORD_ALIGN>(dst, dst_start);
 
     if (dst.fail()) {

lib/libZRCola/build/libZRCola.vcxproj

@@ -64,6 +64,7 @@
   <ItemGroup>
     <ClCompile Include="..\src\character.cpp" />
     <ClCompile Include="..\src\common.cpp" />
+    <ClCompile Include="..\src\highlight.cpp" />
     <ClCompile Include="..\src\language.cpp" />
     <ClCompile Include="..\src\mapping.cpp" />
     <ClCompile Include="..\src\pch.cpp">
@@ -75,6 +76,7 @@
   <ItemGroup>
     <ClInclude Include="..\include\zrcola\character.h" />
     <ClInclude Include="..\include\zrcola\common.h" />
+    <ClInclude Include="..\include\zrcola\highlight.h" />
     <ClInclude Include="..\include\zrcola\language.h" />
     <ClInclude Include="..\include\zrcola\tag.h" />
     <ClInclude Include="..\include\zrcola\translate.h" />

lib/libZRCola/build/libZRCola.vcxproj.filters

@@ -32,6 +32,9 @@
     <ClCompile Include="..\src\tag.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\src\highlight.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\src\pch.h">
@@ -52,5 +55,8 @@
     <ClInclude Include="..\include\zrcola\tag.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="..\include\zrcola\highlight.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
 </Project>

lib/libZRCola/include/zrcola/highlight.h

@@ -0,0 +1,240 @@
+/*
+    SPDX-License-Identifier: GPL-3.0-or-later
+    Copyright © 2021 Amebis
+*/
+
+#pragma once
+
+#include "common.h"
+
+#include <stdex/idrec.h>
+
+#include <functional>
+
+#pragma warning(push)
+#pragma warning(disable: 4200)
+
+///
+/// Default highlight
+///
+#define ZRCOLA_HLGHTSETID_DEFAULT                           ((ZRCola::hlghtsetid_t)0x0000)
+
+
+///
+/// ZRCola Unicode Composed Issues
+///
+#define ZRCOLA_HLGHTSETID_ZRCOLA_UNICODE_COMPOSED_ISSUES    ((ZRCola::hlghtsetid_t)0x0001)
+
+namespace ZRCola {
+    ///
+    /// Highlight set ID
+    ///
+    typedef unsigned __int16 hlghtsetid_t;
+
+    ///
+    /// Highlight database
+    ///
+    class highlight_db {
+    public:
+#pragma pack(push)
+#pragma pack(2)
+        ///
+        /// Highlight data
+        ///
+        struct highlight {
+        public:
+            hlghtsetid_t set;           ///< Highlight set ID
+
+        protected:
+            unsigned __int16 chr_to;    ///< Character end in \c data
+            wchar_t data[];             ///< Character
+
+        private:
+            inline highlight(_In_ const highlight &other);
+            inline highlight& operator=(_In_ const highlight &other);
+
+        public:
+            ///
+            /// Constructs the highlight
+            ///
+            /// \param[in] set      Highlight set ID
+            /// \param[in] chr      Character
+            /// \param[in] chr_len  Number of UTF-16 characters in \p chr
+            ///
+            inline highlight(
+                _In_opt_                        hlghtsetid_t  set     = 0,
+                _In_opt_z_count_(chr_len) const wchar_t      *chr     = NULL,
+                _In_opt_                        size_t        chr_len = 0)
+            {
+                this->set    = set;
+                this->chr_to = static_cast<unsigned __int16>(chr_len);
+                if (chr && chr_len) memcpy(this->data, chr, sizeof(wchar_t)*chr_len);
+            }
+
+            inline const wchar_t*         chr    () const { return data;          };
+            inline       wchar_t*         chr    ()       { return data;          };
+            inline const wchar_t*         chr_end() const { return data + chr_to; };
+            inline       wchar_t*         chr_end()       { return data + chr_to; };
+            inline       unsigned __int16 chr_len() const { return chr_to;        };
+
+            inline wchar_t chr_at(_In_ size_t i) const
+            {
+                return i < chr_to ? data[i] : 0;
+            }
+        };
+#pragma pack(pop)
+
+        ///
+        /// Highlight index
+        ///
+        class indexChr : public index<unsigned __int16, unsigned __int32, highlight>
+        {
+        public:
+            ///
+            /// Constructs the index
+            ///
+            /// \param[in] h  Reference to vector holding the data
+            ///
+            indexChr(_In_ std::vector<unsigned __int16> &h) : index<unsigned __int16, unsigned __int32, highlight>(h) {}
+
+            ///
+            /// Compares two highlights by string (for searching)
+            ///
+            /// \param[in] a  Pointer to first element
+            /// \param[in] b  Pointer to second element
+            ///
+            /// \returns
+            /// - <0 when a <  b
+            /// - =0 when a == b
+            /// - >0 when a >  b
+            ///
+            virtual int compare(_In_ const highlight &a, _In_ const highlight &b) const
+            {
+                int r = ZRCola::CompareString(a.chr(), a.chr_len(), b.chr(), b.chr_len());
+                if (r != 0) return r;
+
+                     if (a.set < b.set) return -1;
+                else if (a.set > b.set) return +1;
+
+                return 0;
+            }
+
+            ///
+            /// Compares two highlights by string (for sorting)
+            ///
+            /// \param[in] a  Pointer to first element
+            /// \param[in] b  Pointer to second element
+            ///
+            /// \returns
+            /// - <0 when a <  b
+            /// - =0 when a == b
+            /// - >0 when a >  b
+            ///
+            virtual int compare_sort(_In_ const highlight &a, _In_ const highlight &b) const
+            {
+                // Revert to `compare()` by default.
+                return compare(a, b);
+            }
+        } idxChr;   ///< Highlight index
+
+
+        std::vector<unsigned __int16> data;         ///< Highlight data
+
+    public:
+        ///
+        /// Constructs the database
+        ///
+        inline highlight_db() : idxChr(data) {}
+
+        ///
+        /// Clears the database
+        ///
+        inline void clear()
+        {
+            idxChr.clear();
+            data  .clear();
+        }
+
+        ///
+        /// Highlights string
+        ///
+        /// \param[in] input     Input string (UTF-16)
+        /// \param[in] inputMax  Length of the input string in characters. Can be (size_t)-1 if \p input is zero terminated.
+        /// \param[in] callback  Function to be called on highlight switch
+        ///
+        void Highlight(_In_z_count_(inputMax) const wchar_t* input, _In_ size_t inputMax, _In_ std::function<void (hlghtsetid_t set, size_t start, size_t end)> callback) const;
+    };
+
+
+    typedef stdex::idrec::record<highlight_db, recordid_t, recordsize_t, ZRCOLA_RECORD_ALIGN> highlight_rec;
+};
+
+
+const ZRCola::recordid_t ZRCola::highlight_rec::id = *(ZRCola::recordid_t*)"HGH";
+
+
+///
+/// Writes highlight database to a stream
+///
+/// \param[in] stream  Output stream
+/// \param[in] db      Highlight database
+///
+/// \returns The stream \p stream
+///
+inline std::ostream& operator <<(_In_ std::ostream& stream, _In_ const ZRCola::highlight_db &db)
+{
+    // Write highlight index.
+    if (stream.fail()) return stream;
+    stream << db.idxChr;
+
+    // Write data count.
+    auto data_count = db.data.size();
+#if defined(_WIN64) || defined(__x86_64__) || defined(__ppc64__)
+    // 4G check
+    if (data_count > 0xffffffff) {
+        stream.setstate(std::ios_base::failbit);
+        return stream;
+    }
+#endif
+    if (stream.fail()) return stream;
+    unsigned __int32 count = (unsigned __int32)data_count;
+    stream.write((const char*)&count, sizeof(count));
+
+    // Write data.
+    if (stream.fail()) return stream;
+    stream.write((const char*)db.data.data(), sizeof(unsigned __int16)*static_cast<std::streamsize>(count));
+
+    return stream;
+}
+
+
+///
+/// Reads highlight database from a stream
+///
+/// \param[in ] stream  Input stream
+/// \param[out] db      Highlight database
+///
+/// \returns The stream \p stream
+///
+inline std::istream& operator >>(_In_ std::istream& stream, _Out_ ZRCola::highlight_db &db)
+{
+    // Read highlight index.
+    stream >> db.idxChr;
+    if (!stream.good()) return stream;
+
+    // Read data count.
+    unsigned __int32 count;
+    stream.read((char*)&count, sizeof(count));
+    if (!stream.good()) return stream;
+
+    if (count) {
+        // Read data.
+        db.data.resize(count);
+        stream.read((char*)db.data.data(), sizeof(unsigned __int16)*static_cast<std::streamsize>(count));
+    } else
+        db.data.clear();
+
+    return stream;
+}
+
+#pragma warning(pop)

lib/libZRCola/src/highlight.cpp

@@ -0,0 +1,77 @@
+/*
+    SPDX-License-Identifier: GPL-3.0-or-later
+    Copyright © 2021 Amebis
+*/
+
+#include "pch.h"
+
+_Use_decl_annotations_
+void ZRCola::highlight_db::Highlight(const wchar_t* input, size_t inputMax, std::function<void (hlghtsetid_t set, size_t start, size_t end)> callback) const
+{
+    size_t start = 0;
+    hlghtsetid_t set = ZRCOLA_HLGHTSETID_DEFAULT;
+
+    for (size_t i = 0; i < inputMax;) {
+        // Find the longest matching highlight at i-th character.
+        size_t l_match = (size_t)-1;
+        for (size_t l = 0, r = idxChr.size(), ii = i, j = 0; ii < inputMax && l < r; ii++, j++) {
+            wchar_t c = input[ii];
+            while (l < r) {
+                // Test the highlight in the middle of the search area.
+                size_t m = (l + r) / 2;
+
+                // Get the j-th character of the highlight.
+                // All highlights that get short on characters are lexically ordered before.
+                // Thus the j-th character is considered 0.
+                wchar_t s = idxChr[m].chr_at(j);
+
+                // Do the bisection test.
+                     if (c < s) r = m;
+                else if (s < c) l = m + 1;
+                else {
+                    // Character found.
+
+                    // Narrow the search area on the left to start at the first highlight in the run.
+                    for (size_t r2 = m; l < r2;) {
+                        size_t m2 = (l + r2) / 2;
+                        if (c <= idxChr[m2].chr_at(j)) r2 = m2; else l = m2 + 1;
+                    }
+
+                    // Narrow the search area on the right to end at the first highlight not in the run.
+                    for (size_t l2 = m + 1; l2 < r;) {
+                        size_t m2 = (l2 + r) / 2;
+                        if (idxChr[m2].chr_at(j) <= c) l2 = m2 + 1; else r = m2;
+                    }
+
+                    if (j + 1 == idxChr[l].chr_len()) {
+                        // The first highlight of the run was a match (thus far). Save it.
+                        l_match = l;
+                    }
+
+                    break;
+                }
+            }
+        }
+
+        if (l_match < idxChr.size()) {
+            // The saved highlight was an exact match.
+            const highlight &hghl = idxChr[l_match];
+            if (set != hghl.set) {
+                callback(set, start, i);
+                start = i;
+                set   = hghl.set;
+            }
+            i += hghl.chr_len();
+        } else {
+            // The match was not found.
+            if (set != ZRCOLA_HLGHTSETID_DEFAULT) {
+                callback(set, start, i);
+                start = i;
+                set   = ZRCOLA_HLGHTSETID_DEFAULT;
+            }
+            i++;
+        }
+    }
+
+    callback(set, start, inputMax);
+}

lib/libZRCola/src/pch.h

@@ -8,6 +8,7 @@
 #include "../../../include/version.h"
 
 #include "../include/zrcola/character.h"
+#include "../include/zrcola/highlight.h"
 #include "../include/zrcola/language.h"
 #include "../include/zrcola/translate.h"
 #include "../include/zrcola/tag.h"