diff --git a/ZRCola/locale/ZRCola.pot b/ZRCola/locale/ZRCola.pot index 08fbd3e..165b5d2 100644 --- a/ZRCola/locale/ZRCola.pot +++ b/ZRCola/locale/ZRCola.pot @@ -2,7 +2,7 @@ msgid "" msgstr "" "Project-Id-Version: ZRCola\n" -"POT-Creation-Date: 2021-12-20 20:17+0100\n" +"POT-Creation-Date: 2021-12-21 14:16+0100\n" "PO-Revision-Date: 2019-04-01 19:38+0200\n" "Last-Translator: Simon Rozman \n" "Language-Team: Amebis, d. o. o., Kamnik \n" @@ -108,36 +108,37 @@ msgstr "Prekini sestavljanje in vrni fokus nazaj izvornemu oknu" #: res/zrcolagui.cpp:138 zrcolagui.cpp:138 #, fuzzy msgid "(De)&composition" -msgstr "Vklopi/izklopi (raz-)sestavljanje ZRCola" +msgstr "(&Raz-)sestavljanje" #: res/zrcolagui.cpp:139 zrcolagui.cpp:139 #, fuzzy msgid "&None" -msgstr "(brez)" +msgstr "&Brez" #: res/zrcolagui.cpp:139 zrcolagui.cpp:139 #, fuzzy msgid "No character (De)composition" -msgstr "Vklopi/izklopi (raz-)sestavljanje ZRCola" +msgstr "Brez (raz-)sestavljanja znakov" #: res/zrcolagui.cpp:142 zrcolagui.cpp:142 #, fuzzy msgid "&ZRCola" -msgstr "ZRCola" +msgstr "&ZRCola" #: res/zrcolagui.cpp:142 zrcolagui.cpp:142 #, fuzzy msgid "ZRCola character (De)composition" -msgstr "Vklopi/izklopi (raz-)sestavljanje ZRCola" +msgstr "(Raz-)sestavljanje znakov ZRCola" #: res/zrcolagui.cpp:145 zrcolagui.cpp:145 #, fuzzy msgid "&Unicode" -msgstr "Unicode" +msgstr "&Unicode" #: res/zrcolagui.cpp:145 zrcolagui.cpp:145 +#, fuzzy msgid "Unicode character (De)composition" -msgstr "" +msgstr "(Raz-)sestavljanje znakov Unicode" #: res/zrcolagui.cpp:151 zrcolagui.cpp:151 #, fuzzy @@ -308,7 +309,7 @@ msgstr "Pošlji razstavljeno" #: res/zrcolagui.cpp:232 zrcolagui.cpp:232 #, fuzzy msgid "No (De)composition" -msgstr "Prekini raz/sestavljanje" +msgstr "Brez (raz-)sestavljanja" #: res/zrcolagui.cpp:232 res/zrcolagui.cpp:940 res/zrcolagui.h:119 #: zrcolaapp.cpp:60 zrcolafrm.cpp:118 zrcolagui.cpp:232 zrcolagui.cpp:943 @@ -807,12 +808,12 @@ msgstr "Opozorilo" msgid "ZRCola keyboard shortcut Win+F6 could not be registered. Some functionality will not be available." msgstr "ZRColine bližnjice na tipkovnici Win+F6 ni mogoče registrirati. Nekaj funkcionalnosti ne bo na voljo." -#: zrcolafrm.cpp:508 +#: zrcolafrm.cpp:513 #, fuzzy msgid "http://zrcola.zrc-sazu.si/en/info/instructions/" msgstr "http://zrcola.zrc-sazu.si/info/instructions/" -#: zrcolafrm.cpp:533 +#: zrcolafrm.cpp:538 #, fuzzy msgid "http://zrcola.zrc-sazu.si/wp-content/uploads/2016/06/ZRCola_tipkovnica_Jun2016.pdf" msgstr "http://zrcola.zrc-sazu.si/wp-content/uploads/2016/06/ZRCola_tipkovnica_Jun2016.pdf" diff --git a/ZRCola/locale/de_DE.po b/ZRCola/locale/de_DE.po index 7397bce..3680d4b 100644 --- a/ZRCola/locale/de_DE.po +++ b/ZRCola/locale/de_DE.po @@ -4,8 +4,8 @@ msgid "" msgstr "" "Project-Id-Version: ZRCola\n" -"POT-Creation-Date: 2021-12-20 20:18+0100\n" -"PO-Revision-Date: 2021-12-20 20:18+0100\n" +"POT-Creation-Date: 2021-12-21 14:17+0100\n" +"PO-Revision-Date: 2021-12-21 14:17+0100\n" "Last-Translator: Simon Rozman \n" "Language-Team: German (Germany) (https://www.transifex.com/amebis/teams/91592/de_DE/)\n" "Language: de_DE\n" @@ -655,11 +655,11 @@ msgstr "" msgid "ZRCola keyboard shortcut Win+F6 could not be registered. Some functionality will not be available." msgstr "" -#: zrcolafrm.cpp:508 +#: zrcolafrm.cpp:513 msgid "http://zrcola.zrc-sazu.si/en/info/instructions/" msgstr "http://zrcola.zrc-sazu.si/de/info/instructions/" -#: zrcolafrm.cpp:533 +#: zrcolafrm.cpp:538 msgid "http://zrcola.zrc-sazu.si/wp-content/uploads/2016/06/ZRCola_tipkovnica_Jun2016.pdf" msgstr "http://zrcola.zrc-sazu.si/wp-content/uploads/2016/06/ZRCola_tipkovnica_Jun2016.pdf" diff --git a/ZRCola/locale/ru_RU.po b/ZRCola/locale/ru_RU.po index b0bb8ae..2ce3017 100644 --- a/ZRCola/locale/ru_RU.po +++ b/ZRCola/locale/ru_RU.po @@ -5,8 +5,8 @@ msgid "" msgstr "" "Project-Id-Version: ZRCola\n" -"POT-Creation-Date: 2021-12-20 20:18+0100\n" -"PO-Revision-Date: 2021-12-20 20:18+0100\n" +"POT-Creation-Date: 2021-12-21 14:17+0100\n" +"PO-Revision-Date: 2021-12-21 14:17+0100\n" "Last-Translator: Simon Rozman \n" "Language-Team: Russian (Russia) (https://www.transifex.com/amebis/teams/91592/ru_RU/)\n" "Language: ru_RU\n" @@ -687,11 +687,11 @@ msgstr "Предупреждение" msgid "ZRCola keyboard shortcut Win+F6 could not be registered. Some functionality will not be available." msgstr "Сочетание клавиш Win+F6 невозможно регистрировать. Некоторые функциональности не будут доступны." -#: zrcolafrm.cpp:508 +#: zrcolafrm.cpp:513 msgid "http://zrcola.zrc-sazu.si/en/info/instructions/" msgstr "http://zrcola.zrc-sazu.si/ru/info/instructions/" -#: zrcolafrm.cpp:533 +#: zrcolafrm.cpp:538 msgid "http://zrcola.zrc-sazu.si/wp-content/uploads/2016/06/ZRCola_tipkovnica_Jun2016.pdf" msgstr "http://zrcola.zrc-sazu.si/wp-content/uploads/2016/06/ZRCola_tipkovnica_Jun2016.pdf" diff --git a/ZRCola/locale/sl_SI.po b/ZRCola/locale/sl_SI.po index 08ec7fe..6d36440 100644 --- a/ZRCola/locale/sl_SI.po +++ b/ZRCola/locale/sl_SI.po @@ -4,8 +4,8 @@ msgid "" msgstr "" "Project-Id-Version: ZRCola\n" -"POT-Creation-Date: 2021-12-20 20:18+0100\n" -"PO-Revision-Date: 2021-12-20 20:20+0100\n" +"POT-Creation-Date: 2021-12-21 14:17+0100\n" +"PO-Revision-Date: 2021-12-21 14:17+0100\n" "Last-Translator: Simon Rozman \n" "Language-Team: Slovenian (Slovenia) (https://www.transifex.com/amebis/teams/91592/sl_SI/)\n" "Language: sl_SI\n" @@ -666,11 +666,11 @@ msgstr "Opozorilo" msgid "ZRCola keyboard shortcut Win+F6 could not be registered. Some functionality will not be available." msgstr "ZRColine bližnjice na tipkovnici Win+F6 ni mogoče registrirati. Nekaj funkcionalnosti ne bo na voljo." -#: zrcolafrm.cpp:508 +#: zrcolafrm.cpp:513 msgid "http://zrcola.zrc-sazu.si/en/info/instructions/" msgstr "http://zrcola.zrc-sazu.si/info/instructions/" -#: zrcolafrm.cpp:533 +#: zrcolafrm.cpp:538 msgid "http://zrcola.zrc-sazu.si/wp-content/uploads/2016/06/ZRCola_tipkovnica_Jun2016.pdf" msgstr "http://zrcola.zrc-sazu.si/wp-content/uploads/2016/06/ZRCola_tipkovnica_Jun2016.pdf" diff --git a/ZRCola/zrcolaapp.cpp b/ZRCola/zrcolaapp.cpp index 7d193f4..bc5646f 100644 --- a/ZRCola/zrcolaapp.cpp +++ b/ZRCola/zrcolaapp.cpp @@ -144,6 +144,12 @@ bool ZRColaApp::OnInit() wxFAIL_MSG(wxT("Error reading tag name data from ZRCola.zrcdb.")); m_tn_db.clear(); } + } else if (id == ZRCola::highlight_rec::id) { + dat >> ZRCola::highlight_rec(m_h_db); + if (!dat.good()) { + wxFAIL_MSG(wxT("Error reading highlight data from ZRCola.zrcdb.")); + m_h_db.clear(); + } } else stdex::idrec::ignore(dat); } diff --git a/ZRCola/zrcolaapp.h b/ZRCola/zrcolaapp.h index 66a28ed..954c573 100644 --- a/ZRCola/zrcolaapp.h +++ b/ZRCola/zrcolaapp.h @@ -21,6 +21,7 @@ class ZRColaApp; #include #pragma warning(pop) #include +#include #include #include #include @@ -72,6 +73,7 @@ public: ZRCola::chrcat_db m_cc_db; ///< Characted category database ZRCola::chrtag_db m_ct_db; ///< Character tag database ZRCola::tagname_db m_tn_db; ///< Tag name database + ZRCola::highlight_db m_h_db; ///< Highlight database wxZRColaFrame *m_mainWnd; ///< Main window diff --git a/ZRCola/zrcolacomppnl.cpp b/ZRCola/zrcolacomppnl.cpp index 8049357..11a5290 100644 --- a/ZRCola/zrcolacomppnl.cpp +++ b/ZRCola/zrcolacomppnl.cpp @@ -17,6 +17,7 @@ wxZRColaComposerPanel::wxZRColaComposerPanel(wxWindow* parent) : m_destinationRestyled(false), m_styleNormal(*wxBLACK, *wxWHITE, wxFont(20, wxFONTFAMILY_DEFAULT, wxFONTSTYLE_NORMAL, wxFONTWEIGHT_NORMAL, false, wxT("ZRCola"))), m_stylePUA(*wxBLUE, *wxWHITE, wxFont(20, wxFONTFAMILY_DEFAULT, wxFONTSTYLE_NORMAL, wxFONTWEIGHT_NORMAL, false, wxT("ZRCola"))), + m_styleZRColaUnicodeComposedIssues(*wxRED, *wxWHITE, wxFont(20, wxFONTFAMILY_DEFAULT, wxFONTSTYLE_NORMAL, wxFONTWEIGHT_NORMAL, false, wxT("ZRCola"))), m_selSource(0, 0), m_selDestination(0, 0), wxZRColaComposerPanelBase(parent) @@ -320,17 +321,25 @@ void wxZRColaComposerPanel::OnDestinationText(wxCommandEvent& event) auto app = dynamic_cast(wxTheApp); m_destinationRestyled = true; - if (app->m_mainWnd->m_warnPUA) { - wxString src = m_destination->GetValue(); - size_t len = src.Length(); - for (size_t i = 0, j; i < len;) { - bool pua_i = ZRCola::ispua(src[i]); - for (j = i + 1; j < len && pua_i == ZRCola::ispua(src[j]); j++); - m_destination->SetStyle((long)i, (long)j, pua_i ? m_stylePUA : m_styleNormal); - i = j; + wxString src = m_destination->GetValue(); + app->m_h_db.Highlight(src, src.Length(), [this, app, src](ZRCola::hlghtsetid_t set, size_t start, size_t end) { + switch (set) { + case ZRCOLA_HLGHTSETID_ZRCOLA_UNICODE_COMPOSED_ISSUES: + m_destination->SetStyle((long)start, (long)end, m_styleZRColaUnicodeComposedIssues); + break; + + default: + if (app->m_mainWnd->m_warnPUA) { + for (size_t i = start, j; i < end;) { + bool pua_i = ZRCola::ispua(src[i]); + for (j = i + 1; j < end && pua_i == ZRCola::ispua(src[j]); j++); + m_destination->SetStyle((long)i, (long)j, pua_i ? m_stylePUA : m_styleNormal); + i = j; + } + } else + m_destination->SetStyle((long)start, (long)end, m_styleNormal); } - } else - m_destination->SetStyle(0, GetWindowTextLength(m_destination->GetHWND()), m_styleNormal); + }); m_destinationRestyled = false; } diff --git a/ZRCola/zrcolacomppnl.h b/ZRCola/zrcolacomppnl.h index 6811ae3..42d7333 100644 --- a/ZRCola/zrcolacomppnl.h +++ b/ZRCola/zrcolacomppnl.h @@ -63,7 +63,8 @@ protected: m_destinationRestyled; ///< Boolean flag to mark destination text is being restyled wxTextAttr m_styleNormal, ///< Normal text style - m_stylePUA; ///< PUA character text style + m_stylePUA, ///< PUA character text style + m_styleZRColaUnicodeComposedIssues; ///< ZRCola Unicode Composed issues character text style std::vector m_mapping; ///< Character index mapping vector between source and normalized text std::pair m_selSource, ///< Character index of selected text in source text control diff --git a/ZRColaCompile/dbsource.cpp b/ZRColaCompile/dbsource.cpp index c67a37e..361abb0 100644 --- a/ZRColaCompile/dbsource.cpp +++ b/ZRColaCompile/dbsource.cpp @@ -242,6 +242,8 @@ ZRCola::DBSource::~DBSource() m_comTranslation.free(); m_pCharacterGroup1.free(); m_comCharacterGroup.free(); + m_pHighlight1.free(); + m_comHighlight.free(); if (m_db) m_db->Close(); @@ -326,6 +328,23 @@ bool ZRCola::DBSource::Open(LPCTSTR filename) wxVERIFY(SUCCEEDED(params->Append(m_pTranslationSets1))); } + wxASSERT_MSG(!m_comHighlight, wxT("ADO command already created")); + wxVERIFY(SUCCEEDED(::CoCreateInstance(CLSID_CADOCommand, NULL, CLSCTX_ALL, IID_IADOCommand, (LPVOID*)&m_comHighlight))); + wxVERIFY(SUCCEEDED(m_comHighlight->put_ActiveConnection(variant(m_db)))); + wxVERIFY(SUCCEEDED(m_comHighlight->put_CommandType(adCmdText))); + wxVERIFY(SUCCEEDED(m_comHighlight->put_CommandText(bstr(L"SELECT [komb] " + L"FROM [VRS_HighlightChars2] " + L"WHERE [group]=? " + L"ORDER BY [komb]")))); + { + // Create and add command parameters. + com_obj params; + wxVERIFY(SUCCEEDED(m_comHighlight->get_Parameters(¶ms))); + wxASSERT_MSG(!m_pHighlight1, wxT("ADO command parameter already created")); + wxVERIFY(SUCCEEDED(m_comHighlight->CreateParameter(bstr(L"@group"), adSmallInt, adParamInput, 0, variant(DISP_E_PARAMNOTFOUND, VT_ERROR), &m_pHighlight1))); + wxVERIFY(SUCCEEDED(params->Append(m_pHighlight1))); + } + return true; } else { _ftprintf(stderr, wxT("%s: error ZCC0011: Could not open database (0x%x).\n"), (LPCTSTR)filename, hr); @@ -1439,3 +1458,41 @@ bool ZRCola::DBSource::GetTagName(const winstd::com_obj& rs, tagna return true; } + + +bool ZRCola::DBSource::SelectHighlights(short set, winstd::com_obj& rs) const +{ + // Create a new recordset. + rs.free(); + wxVERIFY(SUCCEEDED(::CoCreateInstance(CLSID_CADORecordset, NULL, CLSCTX_ALL, IID_IADORecordset, (LPVOID*)&rs))); + wxVERIFY(SUCCEEDED(rs->put_CursorLocation(adUseClient))); + wxVERIFY(SUCCEEDED(rs->put_CursorType(adOpenForwardOnly))); + wxVERIFY(SUCCEEDED(rs->put_LockType(adLockReadOnly))); + + // Open it. + wxVERIFY(SUCCEEDED(m_pHighlight1->put_Value(variant(set)))); + if (FAILED(rs->Open(variant(m_comHighlight), variant(DISP_E_PARAMNOTFOUND, VT_ERROR)))) { + _ftprintf(stderr, wxT("%s: error ZCC0101: Error loading highlights from database. Please make sure the file is ZRCola.zrc compatible.\n"), m_filename.c_str()); + LogErrors(); + return false; + } + + return true; +} + + +bool ZRCola::DBSource::GetHighlight(const com_obj& rs, ZRCola::DBSource::highlight& h) const +{ + wxASSERT_MSG(rs, wxT("recordset is empty")); + + com_obj flds; + wxVERIFY(SUCCEEDED(rs->get_Fields(&flds))); + + { + com_obj f; + wxVERIFY(SUCCEEDED(flds->get_Item(variant(L"komb"), &f))); + wxCHECK(GetUnicodeString(f, h.chr), false); + } + + return true; +} diff --git a/ZRColaCompile/dbsource.h b/ZRColaCompile/dbsource.h index 80c490d..d176c59 100644 --- a/ZRColaCompile/dbsource.h +++ b/ZRColaCompile/dbsource.h @@ -6,7 +6,7 @@ #pragma once #include -#include +#include #include #include #include @@ -404,6 +404,18 @@ namespace ZRCola { }; + /// + /// Highlight + /// + class highlight { + public: + short set; ///< Highlight set ID + std::wstring chr; ///< Character sequence + + inline highlight() : set((short)ZRCOLA_HLGHTSETID_DEFAULT) {} + }; + + public: DBSource(); virtual ~DBSource(); @@ -452,18 +464,6 @@ namespace ZRCola { return SUCCEEDED(rs->get_RecordCount(&count)) ? count : (size_t)-1; } - /// - /// Splits string to individual keywords - /// - /// \param[in ] str String - /// \param[out] keywords Array of keywords - /// - /// \returns - /// - true when successful - /// - false otherwise - /// - static bool GetKeywords(const wchar_t *str, std::vector< std::wstring > &keywords); - /// /// Gets boolean from ZRCola.zrc database /// @@ -872,6 +872,30 @@ namespace ZRCola { /// bool GetTagName(const winstd::com_obj& rs, tagname& tn) const; + /// + /// Returns character highlights by set + /// + /// \param[in ] set Highlight set ID + /// \param[out] rs Recordset with results + /// + /// \returns + /// - true when query succeeds + /// - false otherwise + /// + bool SelectHighlights(short set, winstd::com_obj& rs) const; + + /// + /// Returns highlight data + /// + /// \param[in] rs Recordset with results + /// \param[out] h Highlight + /// + /// \returns + /// - true when succeeded + /// - false otherwise + /// + bool GetHighlight(const winstd::com_obj& rs, highlight& h) const; + protected: std::basic_string m_filename; ///< Database filename winstd::com_obj m_db; ///< Database @@ -886,6 +910,9 @@ namespace ZRCola { winstd::com_obj m_comTranslationSets; ///< ADO Command for GetTranslationSeq subquery winstd::com_obj m_pTranslationSets1; ///< \c m_comTranslationSets parameter + winstd::com_obj m_comHighlight; ///< ADO Command for SelectHighlights subquery + winstd::com_obj m_pHighlight1; ///< \c m_comHighlights parameter + std::set m_terms_ignore; ///< Terms to ignore when comparing characters }; }; @@ -1098,3 +1125,17 @@ inline ZRCola::tagname_db& operator<<(_Inout_ ZRCola::tagname_db &db, _In_ const return db; } + + +inline ZRCola::highlight_db& operator<<(_Inout_ ZRCola::highlight_db &db, _In_ const ZRCola::DBSource::highlight &rec) +{ + unsigned __int32 idx = db.data.size(); + db.data.push_back((unsigned __int16)rec.set); + std::wstring::size_type n = rec.chr.length(); + wxASSERT_MSG(n <= 0xffff, wxT("character overflow")); + db.data.push_back((unsigned __int16)n); + db.data.insert(db.data.end(), rec.chr.cbegin(), rec.chr.cend()); + db.idxChr.push_back(idx); + + return db; +} diff --git a/ZRColaCompile/main.cpp b/ZRColaCompile/main.cpp index 7cd3d8b..960e2a7 100644 --- a/ZRColaCompile/main.cpp +++ b/ZRColaCompile/main.cpp @@ -374,9 +374,9 @@ int _tmain(int argc, _TCHAR *argv[]) } // Preallocate memory. - db_trans.idxSrc.reserve(count); - db_trans.idxDst.reserve(count); - db_trans.data .reserve(count*5); + db_trans.idxSrc.reserve(count*2); + db_trans.idxDst.reserve(count*2); + db_trans.data .reserve(count*2*8); // Parse translations and build index and data. ZRCola::DBSource::translation trans; @@ -399,6 +399,7 @@ int _tmain(int argc, _TCHAR *argv[]) } if (!has_pua) { trans.set = (short)ZRCOLA_TRANSETID_UNICODE; + trans.dst.rank += 50; db_trans << trans; } } @@ -416,7 +417,13 @@ int _tmain(int argc, _TCHAR *argv[]) { com_obj rs_tran; if (src.SelectTranslations(static_cast(ZRCOLA_TRANSETID_UNICODE), rs_tran)) { - if (src.GetRecordsetCount(rs_tran) < 0xffffffff) { // 4G check (-1 is reserved for error condition) + size_t count = src.GetRecordsetCount(rs_tran); + if (count < 0xffffffff) { // 4G check (-1 is reserved for error condition) + // Preallocate memory. + db_trans.idxSrc.reserve(db_trans.idxSrc.size() + count); + db_trans.idxDst.reserve(db_trans.idxDst.size() + count); + db_trans.data .reserve(db_trans.data.size() + count*8); + // Parse translations and build temporary database. ZRCola::DBSource::translation trans; trans.set = (short)ZRCOLA_TRANSETID_UNICODE; @@ -467,7 +474,13 @@ int _tmain(int argc, _TCHAR *argv[]) // Get translations. com_obj rs_tran; if (src.SelectTranslations(ts.set, rs_tran)) { - if (src.GetRecordsetCount(rs_tran) < 0xffffffff) { // 4G check (-1 is reserved for error condition) + count = src.GetRecordsetCount(rs_tran); + if (count < 0xffffffff) { // 4G check (-1 is reserved for error condition) + // Preallocate memory. + db_trans.idxSrc.reserve(db_trans.idxSrc.size() + count); + db_trans.idxDst.reserve(db_trans.idxDst.size() + count); + db_trans.data .reserve(db_trans.data.size() + count*8); + // Parse translations and build temporary database. ZRCola::DBSource::translation trans; trans.set = ts.set; @@ -500,17 +513,13 @@ int _tmain(int argc, _TCHAR *argv[]) } } - // Sort indices. - db_transset.idxTranSet.sort(); - // Write translation sets to file. + db_transset.idxTranSet.sort(); dst << ZRCola::transet_rec(db_transset); - // Sort indices. + // Write translations to file. db_trans.idxSrc.sort(); db_trans.idxDst.sort(); - - // Write translations to file. dst << ZRCola::translation_rec(db_trans); { @@ -540,11 +549,9 @@ int _tmain(int argc, _TCHAR *argv[]) has_errors = true; } - // Sort indices. + // Write translation sequences to file. db.idxTranSeq.sort(); db.idxRank .sort(); - - // Write translation sequences to file. dst << ZRCola::transeq_rec(db); } else { _ftprintf(stderr, wxT("%s: error ZCC0025: Error getting translation sequence count from database or too many translation sequences.\n"), (LPCTSTR)filenameIn.c_str()); @@ -639,10 +646,8 @@ int _tmain(int argc, _TCHAR *argv[]) has_errors = true; } - // Sort indices. - db.idxLang.sort(); - // Write languages to file. + db.idxLang.sort(); dst << ZRCola::language_rec(db); } else { _ftprintf(stderr, wxT("%s: error ZCC0009: Error getting language count from database or too many languages.\n"), (LPCTSTR)filenameIn.c_str()); @@ -680,13 +685,11 @@ int _tmain(int argc, _TCHAR *argv[]) has_errors = true; } - // Sort indices. + // Write language characters to file. db.idxChr .sort(); #ifdef ZRCOLA_LANGCHAR_LANG_IDX db.idxLang.sort(); #endif - - // Write language characters to file. dst << ZRCola::langchar_rec(db); } else { _ftprintf(stderr, wxT("%s: error ZCC0011: Error getting language characters count from database or too many langchars.\n"), (LPCTSTR)filenameIn.c_str()); @@ -729,10 +732,8 @@ int _tmain(int argc, _TCHAR *argv[]) has_errors = true; } - // Sort indices. - db.idxRank.sort(); - // Write character groups to file. + db.idxRank.sort(); dst << ZRCola::chrgrp_rec(db); } else { _ftprintf(stderr, wxT("%s: error ZCC0015: Error getting character group count from database or too many character groups.\n"), (LPCTSTR)filenameIn.c_str()); @@ -753,7 +754,6 @@ int _tmain(int argc, _TCHAR *argv[]) size_t count = src.GetRecordsetCount(rs); if (count < 0xffffffff) { // 4G check (-1 is reserved for error condition) ZRCola::DBSource::character_desc_idx idxChrDsc, idxChrDscSub; - ZRCola::DBSource::character_bank chrs; // Phase 1: Parse characters and build indexes. @@ -788,14 +788,10 @@ int _tmain(int argc, _TCHAR *argv[]) categories_used.insert(chr->second.cat); } - // Sort indices. + // Write characters to file. db.idxChr.sort(); - - // Save text indices. idxChrDsc .save(db.idxDsc ); idxChrDscSub.save(db.idxDscSub); - - // Write characters to file. dst << ZRCola::character_rec(db); } else { _ftprintf(stderr, wxT("%s: error ZCC0017: Error getting character count from database or too many characters.\n"), (LPCTSTR)filenameIn.c_str()); @@ -839,11 +835,9 @@ int _tmain(int argc, _TCHAR *argv[]) has_errors = true; } - // Sort indices. + // Write character categories to file. db.idxChrCat.sort(); db.idxRank .sort(); - - // Write character categories to file. dst << ZRCola::chrcat_rec(db); } else { _ftprintf(stderr, wxT("%s: error ZCC0019: Error getting character category count from database or too many character categories.\n"), (LPCTSTR)filenameIn.c_str()); @@ -879,11 +873,9 @@ int _tmain(int argc, _TCHAR *argv[]) has_errors = true; } - // Sort indices. + // Write characters tags to file. db.idxChr.sort(); db.idxTag.sort(); - - // Write characters tags to file. dst << ZRCola::chrtag_rec(db); } else { _ftprintf(stderr, wxT("%s: error ZCC0021: Error getting characters tags count from database or too many character tags.\n"), (LPCTSTR)filenameIn.c_str()); @@ -919,11 +911,9 @@ int _tmain(int argc, _TCHAR *argv[]) has_errors = true; } - // Sort indices. + // Write tags to file. db.idxName.sort(); db.idxTag .sort(); - - // Write tags to file. dst << ZRCola::tagname_rec(db); } else { _ftprintf(stderr, wxT("%s: error ZCC0023: Error getting tag name count from database or too many tags.\n"), (LPCTSTR)filenameIn.c_str()); @@ -935,6 +925,43 @@ int _tmain(int argc, _TCHAR *argv[]) } } + { + // Get highlights. + com_obj rs; + if (src.SelectHighlights((short)ZRCOLA_HLGHTSETID_ZRCOLA_UNICODE_COMPOSED_ISSUES, rs)) { + size_t count = src.GetRecordsetCount(rs); + if (count < 0xffffffff) { // 4G check (-1 is reserved for error condition) + ZRCola::DBSource::highlight h; + ZRCola::highlight_db db; + + // Preallocate memory. + db.idxChr.reserve(count); + db.data .reserve(count*5); + + // Parse highlights and build index and data. + h.set = (short)ZRCOLA_HLGHTSETID_ZRCOLA_UNICODE_COMPOSED_ISSUES; + for (; !ZRCola::DBSource::IsEOF(rs); rs->MoveNext()) { + // Read tag name from the database. + if (src.GetHighlight(rs, h)) { + // Add highlight to index and data. + db << h; + } else + has_errors = true; + } + + // Write highlights to file. + db.idxChr.sort(); + dst << ZRCola::highlight_rec(db); + } else { + _ftprintf(stderr, wxT("%s: error ZCC0027: Error getting highlight count from database or too many tags.\n"), (LPCTSTR)filenameIn.c_str()); + has_errors = true; + } + } else { + _ftprintf(stderr, wxT("%s: error ZCC0026: Error getting highlights from database. Please make sure the file is ZRCola.zrc compatible.\n"), (LPCTSTR)filenameIn.c_str()); + has_errors = true; + } + } + idrec::close(dst, dst_start); if (dst.fail()) { diff --git a/lib/libZRCola/build/libZRCola.vcxproj b/lib/libZRCola/build/libZRCola.vcxproj index f2d89bd..45f4cbd 100644 --- a/lib/libZRCola/build/libZRCola.vcxproj +++ b/lib/libZRCola/build/libZRCola.vcxproj @@ -64,6 +64,7 @@ + @@ -75,6 +76,7 @@ + diff --git a/lib/libZRCola/build/libZRCola.vcxproj.filters b/lib/libZRCola/build/libZRCola.vcxproj.filters index 4546b17..ea574dd 100644 --- a/lib/libZRCola/build/libZRCola.vcxproj.filters +++ b/lib/libZRCola/build/libZRCola.vcxproj.filters @@ -32,6 +32,9 @@ Source Files + + Source Files + @@ -52,5 +55,8 @@ Header Files + + Header Files + \ No newline at end of file diff --git a/lib/libZRCola/include/zrcola/highlight.h b/lib/libZRCola/include/zrcola/highlight.h new file mode 100644 index 0000000..073e9ac --- /dev/null +++ b/lib/libZRCola/include/zrcola/highlight.h @@ -0,0 +1,240 @@ +/* + SPDX-License-Identifier: GPL-3.0-or-later + Copyright © 2021 Amebis +*/ + +#pragma once + +#include "common.h" + +#include + +#include + +#pragma warning(push) +#pragma warning(disable: 4200) + +/// +/// Default highlight +/// +#define ZRCOLA_HLGHTSETID_DEFAULT ((ZRCola::hlghtsetid_t)0x0000) + + +/// +/// ZRCola Unicode Composed Issues +/// +#define ZRCOLA_HLGHTSETID_ZRCOLA_UNICODE_COMPOSED_ISSUES ((ZRCola::hlghtsetid_t)0x0001) + +namespace ZRCola { + /// + /// Highlight set ID + /// + typedef unsigned __int16 hlghtsetid_t; + + /// + /// Highlight database + /// + class highlight_db { + public: +#pragma pack(push) +#pragma pack(2) + /// + /// Highlight data + /// + struct highlight { + public: + hlghtsetid_t set; ///< Highlight set ID + + protected: + unsigned __int16 chr_to; ///< Character end in \c data + wchar_t data[]; ///< Character + + private: + inline highlight(_In_ const highlight &other); + inline highlight& operator=(_In_ const highlight &other); + + public: + /// + /// Constructs the highlight + /// + /// \param[in] set Highlight set ID + /// \param[in] chr Character + /// \param[in] chr_len Number of UTF-16 characters in \p chr + /// + inline highlight( + _In_opt_ hlghtsetid_t set = 0, + _In_opt_z_count_(chr_len) const wchar_t *chr = NULL, + _In_opt_ size_t chr_len = 0) + { + this->set = set; + this->chr_to = static_cast(chr_len); + if (chr && chr_len) memcpy(this->data, chr, sizeof(wchar_t)*chr_len); + } + + inline const wchar_t* chr () const { return data; }; + inline wchar_t* chr () { return data; }; + inline const wchar_t* chr_end() const { return data + chr_to; }; + inline wchar_t* chr_end() { return data + chr_to; }; + inline unsigned __int16 chr_len() const { return chr_to; }; + + inline wchar_t chr_at(_In_ size_t i) const + { + return i < chr_to ? data[i] : 0; + } + }; +#pragma pack(pop) + + /// + /// Highlight index + /// + class indexChr : public index + { + public: + /// + /// Constructs the index + /// + /// \param[in] h Reference to vector holding the data + /// + indexChr(_In_ std::vector &h) : index(h) {} + + /// + /// Compares two highlights by string (for searching) + /// + /// \param[in] a Pointer to first element + /// \param[in] b Pointer to second element + /// + /// \returns + /// - <0 when a < b + /// - =0 when a == b + /// - >0 when a > b + /// + virtual int compare(_In_ const highlight &a, _In_ const highlight &b) const + { + int r = ZRCola::CompareString(a.chr(), a.chr_len(), b.chr(), b.chr_len()); + if (r != 0) return r; + + if (a.set < b.set) return -1; + else if (a.set > b.set) return +1; + + return 0; + } + + /// + /// Compares two highlights by string (for sorting) + /// + /// \param[in] a Pointer to first element + /// \param[in] b Pointer to second element + /// + /// \returns + /// - <0 when a < b + /// - =0 when a == b + /// - >0 when a > b + /// + virtual int compare_sort(_In_ const highlight &a, _In_ const highlight &b) const + { + // Revert to `compare()` by default. + return compare(a, b); + } + } idxChr; ///< Highlight index + + + std::vector data; ///< Highlight data + + public: + /// + /// Constructs the database + /// + inline highlight_db() : idxChr(data) {} + + /// + /// Clears the database + /// + inline void clear() + { + idxChr.clear(); + data .clear(); + } + + /// + /// Highlights string + /// + /// \param[in] input Input string (UTF-16) + /// \param[in] inputMax Length of the input string in characters. Can be (size_t)-1 if \p input is zero terminated. + /// \param[in] callback Function to be called on highlight switch + /// + void Highlight(_In_z_count_(inputMax) const wchar_t* input, _In_ size_t inputMax, _In_ std::function callback) const; + }; + + + typedef stdex::idrec::record highlight_rec; +}; + + +const ZRCola::recordid_t ZRCola::highlight_rec::id = *(ZRCola::recordid_t*)"HGH"; + + +/// +/// Writes highlight database to a stream +/// +/// \param[in] stream Output stream +/// \param[in] db Highlight database +/// +/// \returns The stream \p stream +/// +inline std::ostream& operator <<(_In_ std::ostream& stream, _In_ const ZRCola::highlight_db &db) +{ + // Write highlight index. + if (stream.fail()) return stream; + stream << db.idxChr; + + // Write data count. + auto data_count = db.data.size(); +#if defined(_WIN64) || defined(__x86_64__) || defined(__ppc64__) + // 4G check + if (data_count > 0xffffffff) { + stream.setstate(std::ios_base::failbit); + return stream; + } +#endif + if (stream.fail()) return stream; + unsigned __int32 count = (unsigned __int32)data_count; + stream.write((const char*)&count, sizeof(count)); + + // Write data. + if (stream.fail()) return stream; + stream.write((const char*)db.data.data(), sizeof(unsigned __int16)*static_cast(count)); + + return stream; +} + + +/// +/// Reads highlight database from a stream +/// +/// \param[in ] stream Input stream +/// \param[out] db Highlight database +/// +/// \returns The stream \p stream +/// +inline std::istream& operator >>(_In_ std::istream& stream, _Out_ ZRCola::highlight_db &db) +{ + // Read highlight index. + stream >> db.idxChr; + if (!stream.good()) return stream; + + // Read data count. + unsigned __int32 count; + stream.read((char*)&count, sizeof(count)); + if (!stream.good()) return stream; + + if (count) { + // Read data. + db.data.resize(count); + stream.read((char*)db.data.data(), sizeof(unsigned __int16)*static_cast(count)); + } else + db.data.clear(); + + return stream; +} + +#pragma warning(pop) diff --git a/lib/libZRCola/src/highlight.cpp b/lib/libZRCola/src/highlight.cpp new file mode 100644 index 0000000..59717f2 --- /dev/null +++ b/lib/libZRCola/src/highlight.cpp @@ -0,0 +1,77 @@ +/* + SPDX-License-Identifier: GPL-3.0-or-later + Copyright © 2021 Amebis +*/ + +#include "pch.h" + +_Use_decl_annotations_ +void ZRCola::highlight_db::Highlight(const wchar_t* input, size_t inputMax, std::function callback) const +{ + size_t start = 0; + hlghtsetid_t set = ZRCOLA_HLGHTSETID_DEFAULT; + + for (size_t i = 0; i < inputMax;) { + // Find the longest matching highlight at i-th character. + size_t l_match = (size_t)-1; + for (size_t l = 0, r = idxChr.size(), ii = i, j = 0; ii < inputMax && l < r; ii++, j++) { + wchar_t c = input[ii]; + while (l < r) { + // Test the highlight in the middle of the search area. + size_t m = (l + r) / 2; + + // Get the j-th character of the highlight. + // All highlights that get short on characters are lexically ordered before. + // Thus the j-th character is considered 0. + wchar_t s = idxChr[m].chr_at(j); + + // Do the bisection test. + if (c < s) r = m; + else if (s < c) l = m + 1; + else { + // Character found. + + // Narrow the search area on the left to start at the first highlight in the run. + for (size_t r2 = m; l < r2;) { + size_t m2 = (l + r2) / 2; + if (c <= idxChr[m2].chr_at(j)) r2 = m2; else l = m2 + 1; + } + + // Narrow the search area on the right to end at the first highlight not in the run. + for (size_t l2 = m + 1; l2 < r;) { + size_t m2 = (l2 + r) / 2; + if (idxChr[m2].chr_at(j) <= c) l2 = m2 + 1; else r = m2; + } + + if (j + 1 == idxChr[l].chr_len()) { + // The first highlight of the run was a match (thus far). Save it. + l_match = l; + } + + break; + } + } + } + + if (l_match < idxChr.size()) { + // The saved highlight was an exact match. + const highlight &hghl = idxChr[l_match]; + if (set != hghl.set) { + callback(set, start, i); + start = i; + set = hghl.set; + } + i += hghl.chr_len(); + } else { + // The match was not found. + if (set != ZRCOLA_HLGHTSETID_DEFAULT) { + callback(set, start, i); + start = i; + set = ZRCOLA_HLGHTSETID_DEFAULT; + } + i++; + } + } + + callback(set, start, inputMax); +} diff --git a/lib/libZRCola/src/pch.h b/lib/libZRCola/src/pch.h index 56ddbd9..8787b18 100644 --- a/lib/libZRCola/src/pch.h +++ b/lib/libZRCola/src/pch.h @@ -8,6 +8,7 @@ #include "../../../include/version.h" #include "../include/zrcola/character.h" +#include "../include/zrcola/highlight.h" #include "../include/zrcola/language.h" #include "../include/zrcola/translate.h" #include "../include/zrcola/tag.h" diff --git a/output/data/ZRCola.zrcdb b/output/data/ZRCola.zrcdb index e5dd6ad..d99d0cc 100644 Binary files a/output/data/ZRCola.zrcdb and b/output/data/ZRCola.zrcdb differ