Amebis/ZRCola
Amebis/ZRCola
1
1
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 49 Wiki Activity

Introduce bad ZRCola Unicode Composition highlighting

Browse Source 
Signed-off-by: Simon Rozman <simon@rozman.si>
This commit is contained in:
Simon Rozman 2021-12-21 14:21:28 +01:00
parent 6c3b2635f4
commit 7fb29aed80
17 changed files with 555 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
ZRCola/locale/ZRCola.pot
View File

@ -2,7 +2,7 @@
msgid ""
msgstr ""
"Project-Id-Version: ZRCola\n"
"POT-Creation-Date: 2021-12-20 20:17+0100\n"
"POT-Creation-Date: 2021-12-21 14:16+0100\n"
"PO-Revision-Date: 2019-04-01 19:38+0200\n"
"Last-Translator: Simon Rozman <simon.rozman@amebis.si>\n"
"Language-Team: Amebis, d. o. o., Kamnik <info@amebis.si>\n"
@ -108,36 +108,37 @@ msgstr "Prekini sestavljanje in vrni fokus nazaj izvornemu oknu"
#: res/zrcolagui.cpp:138 zrcolagui.cpp:138
#, fuzzy
msgid "(De)&composition"
msgstr "Vklopi/izklopi (raz-)sestavljanje ZRCola"
msgstr "(&Raz-)sestavljanje"
#: res/zrcolagui.cpp:139 zrcolagui.cpp:139
#, fuzzy
msgid "&None"
msgstr "(brez)"
msgstr "&Brez"
#: res/zrcolagui.cpp:139 zrcolagui.cpp:139
#, fuzzy
msgid "No character (De)composition"
msgstr "Vklopi/izklopi (raz-)sestavljanje ZRCola"
msgstr "Brez (raz-)sestavljanja znakov"
#: res/zrcolagui.cpp:142 zrcolagui.cpp:142
#, fuzzy
msgid "&ZRCola"
msgstr "ZRCola"
msgstr "&ZRCola"
#: res/zrcolagui.cpp:142 zrcolagui.cpp:142
#, fuzzy
msgid "ZRCola character (De)composition"
msgstr "Vklopi/izklopi (raz-)sestavljanje ZRCola"
msgstr "(Raz-)sestavljanje znakov ZRCola"
#: res/zrcolagui.cpp:145 zrcolagui.cpp:145
#, fuzzy
msgid "&Unicode"
msgstr "Unicode"
msgstr "&Unicode"
#: res/zrcolagui.cpp:145 zrcolagui.cpp:145
#, fuzzy
msgid "Unicode character (De)composition"
msgstr ""
msgstr "(Raz-)sestavljanje znakov Unicode"
#: res/zrcolagui.cpp:151 zrcolagui.cpp:151
#, fuzzy
@ -308,7 +309,7 @@ msgstr "Pošlji razstavljeno"
#: res/zrcolagui.cpp:232 zrcolagui.cpp:232
#, fuzzy
msgid "No (De)composition"
msgstr "Prekini raz/sestavljanje"
msgstr "Brez (raz-)sestavljanja"
#: res/zrcolagui.cpp:232 res/zrcolagui.cpp:940 res/zrcolagui.h:119
#: zrcolaapp.cpp:60 zrcolafrm.cpp:118 zrcolagui.cpp:232 zrcolagui.cpp:943
@ -807,12 +808,12 @@ msgstr "Opozorilo"
msgid "ZRCola keyboard shortcut Win+F6 could not be registered. Some functionality will not be available."
msgstr "ZRColine bližnjice na tipkovnici Win+F6 ni mogoče registrirati. Nekaj funkcionalnosti ne bo na voljo."
#: zrcolafrm.cpp:508
#: zrcolafrm.cpp:513
#, fuzzy
msgid "http://zrcola.zrc-sazu.si/en/info/instructions/"
msgstr "http://zrcola.zrc-sazu.si/info/instructions/"
#: zrcolafrm.cpp:533
#: zrcolafrm.cpp:538
#, fuzzy
msgid "http://zrcola.zrc-sazu.si/wp-content/uploads/2016/06/ZRCola_tipkovnica_Jun2016.pdf"
msgstr "http://zrcola.zrc-sazu.si/wp-content/uploads/2016/06/ZRCola_tipkovnica_Jun2016.pdf"

8
ZRCola/locale/de_DE.po
View File

@ -4,8 +4,8 @@
msgid ""
msgstr ""
"Project-Id-Version: ZRCola\n"
"POT-Creation-Date: 2021-12-20 20:18+0100\n"
"PO-Revision-Date: 2021-12-20 20:18+0100\n"
"POT-Creation-Date: 2021-12-21 14:17+0100\n"
"PO-Revision-Date: 2021-12-21 14:17+0100\n"
"Last-Translator: Simon Rozman <simon.rozman@amebis.si>\n"
"Language-Team: German (Germany) (https://www.transifex.com/amebis/teams/91592/de_DE/)\n"
"Language: de_DE\n"
@ -655,11 +655,11 @@ msgstr ""
msgid "ZRCola keyboard shortcut Win+F6 could not be registered. Some functionality will not be available."
msgstr ""
#: zrcolafrm.cpp:508
#: zrcolafrm.cpp:513
msgid "http://zrcola.zrc-sazu.si/en/info/instructions/"
msgstr "http://zrcola.zrc-sazu.si/de/info/instructions/"
#: zrcolafrm.cpp:533
#: zrcolafrm.cpp:538
msgid "http://zrcola.zrc-sazu.si/wp-content/uploads/2016/06/ZRCola_tipkovnica_Jun2016.pdf"
msgstr "http://zrcola.zrc-sazu.si/wp-content/uploads/2016/06/ZRCola_tipkovnica_Jun2016.pdf"

8
ZRCola/locale/ru_RU.po
View File

@ -5,8 +5,8 @@
msgid ""
msgstr ""
"Project-Id-Version: ZRCola\n"
"POT-Creation-Date: 2021-12-20 20:18+0100\n"
"PO-Revision-Date: 2021-12-20 20:18+0100\n"
"POT-Creation-Date: 2021-12-21 14:17+0100\n"
"PO-Revision-Date: 2021-12-21 14:17+0100\n"
"Last-Translator: Simon Rozman <simon.rozman@amebis.si>\n"
"Language-Team: Russian (Russia) (https://www.transifex.com/amebis/teams/91592/ru_RU/)\n"
"Language: ru_RU\n"
@ -687,11 +687,11 @@ msgstr "Предупреждение"
msgid "ZRCola keyboard shortcut Win+F6 could not be registered. Some functionality will not be available."
msgstr "Сочетание клавиш Win+F6 невозможно регистрировать. Некоторые функциональности не будут доступны."
#: zrcolafrm.cpp:508
#: zrcolafrm.cpp:513
msgid "http://zrcola.zrc-sazu.si/en/info/instructions/"
msgstr "http://zrcola.zrc-sazu.si/ru/info/instructions/"
#: zrcolafrm.cpp:533
#: zrcolafrm.cpp:538
msgid "http://zrcola.zrc-sazu.si/wp-content/uploads/2016/06/ZRCola_tipkovnica_Jun2016.pdf"
msgstr "http://zrcola.zrc-sazu.si/wp-content/uploads/2016/06/ZRCola_tipkovnica_Jun2016.pdf"

8
ZRCola/locale/sl_SI.po
View File

@ -4,8 +4,8 @@
msgid ""
msgstr ""
"Project-Id-Version: ZRCola\n"
"POT-Creation-Date: 2021-12-20 20:18+0100\n"
"PO-Revision-Date: 2021-12-20 20:20+0100\n"
"POT-Creation-Date: 2021-12-21 14:17+0100\n"
"PO-Revision-Date: 2021-12-21 14:17+0100\n"
"Last-Translator: Simon Rozman <simon.rozman@amebis.si>\n"
"Language-Team: Slovenian (Slovenia) (https://www.transifex.com/amebis/teams/91592/sl_SI/)\n"
"Language: sl_SI\n"
@ -666,11 +666,11 @@ msgstr "Opozorilo"
msgid "ZRCola keyboard shortcut Win+F6 could not be registered. Some functionality will not be available."
msgstr "ZRColine bližnjice na tipkovnici Win+F6 ni mogoče registrirati. Nekaj funkcionalnosti ne bo na voljo."
#: zrcolafrm.cpp:508
#: zrcolafrm.cpp:513
msgid "http://zrcola.zrc-sazu.si/en/info/instructions/"
msgstr "http://zrcola.zrc-sazu.si/info/instructions/"
#: zrcolafrm.cpp:533
#: zrcolafrm.cpp:538
msgid "http://zrcola.zrc-sazu.si/wp-content/uploads/2016/06/ZRCola_tipkovnica_Jun2016.pdf"
msgstr "http://zrcola.zrc-sazu.si/wp-content/uploads/2016/06/ZRCola_tipkovnica_Jun2016.pdf"

6
ZRCola/zrcolaapp.cpp
View File

@ -144,6 +144,12 @@ bool ZRColaApp::OnInit()
wxFAIL_MSG(wxT("Error reading tag name data from ZRCola.zrcdb."));
m_tn_db.clear();
}
} else if (id == ZRCola::highlight_rec::id) {
dat >> ZRCola::highlight_rec(m_h_db);
if (!dat.good()) {
wxFAIL_MSG(wxT("Error reading highlight data from ZRCola.zrcdb."));
m_h_db.clear();
}
} else
stdex::idrec::ignore<ZRCola::recordsize_t, ZRCOLA_RECORD_ALIGN>(dat);
}

2
ZRCola/zrcolaapp.h
View File

@ -21,6 +21,7 @@ class ZRColaApp;
#include <wx/intl.h>
#pragma warning(pop)
#include <zrcola/character.h>
#include <zrcola/highlight.h>
#include <zrcola/language.h>
#include <zrcola/translate.h>
#include <zrcola/tag.h>
@ -72,6 +73,7 @@ public:
ZRCola::chrcat_db m_cc_db; ///< Characted category database
ZRCola::chrtag_db m_ct_db; ///< Character tag database
ZRCola::tagname_db m_tn_db; ///< Tag name database
ZRCola::highlight_db m_h_db; ///< Highlight database
wxZRColaFrame *m_mainWnd; ///< Main window

19
ZRCola/zrcolacomppnl.cpp
View File

@ -17,6 +17,7 @@ wxZRColaComposerPanel::wxZRColaComposerPanel(wxWindow* parent) :
m_destinationRestyled(false),
m_styleNormal(*wxBLACK, *wxWHITE, wxFont(20, wxFONTFAMILY_DEFAULT, wxFONTSTYLE_NORMAL, wxFONTWEIGHT_NORMAL, false, wxT("ZRCola"))),
m_stylePUA(*wxBLUE, *wxWHITE, wxFont(20, wxFONTFAMILY_DEFAULT, wxFONTSTYLE_NORMAL, wxFONTWEIGHT_NORMAL, false, wxT("ZRCola"))),
m_styleZRColaUnicodeComposedIssues(*wxRED, *wxWHITE, wxFont(20, wxFONTFAMILY_DEFAULT, wxFONTSTYLE_NORMAL, wxFONTWEIGHT_NORMAL, false, wxT("ZRCola"))),
m_selSource(0, 0),
m_selDestination(0, 0),
wxZRColaComposerPanelBase(parent)
@ -320,17 +321,25 @@ void wxZRColaComposerPanel::OnDestinationText(wxCommandEvent& event)
auto app = dynamic_cast<ZRColaApp*>(wxTheApp);
m_destinationRestyled = true;
if (app->m_mainWnd->m_warnPUA) {
wxString src = m_destination->GetValue();
size_t len = src.Length();
for (size_t i = 0, j; i < len;) {
app->m_h_db.Highlight(src, src.Length(), [this, app, src](ZRCola::hlghtsetid_t set, size_t start, size_t end) {
switch (set) {
case ZRCOLA_HLGHTSETID_ZRCOLA_UNICODE_COMPOSED_ISSUES:
m_destination->SetStyle((long)start, (long)end, m_styleZRColaUnicodeComposedIssues);
break;
default:
if (app->m_mainWnd->m_warnPUA) {
for (size_t i = start, j; i < end;) {
bool pua_i = ZRCola::ispua(src[i]);
for (j = i + 1; j < len && pua_i == ZRCola::ispua(src[j]); j++);
for (j = i + 1; j < end && pua_i == ZRCola::ispua(src[j]); j++);
m_destination->SetStyle((long)i, (long)j, pua_i ? m_stylePUA : m_styleNormal);
i = j;
}
} else
m_destination->SetStyle(0, GetWindowTextLength(m_destination->GetHWND()), m_styleNormal);
m_destination->SetStyle((long)start, (long)end, m_styleNormal);
}
});
m_destinationRestyled = false;
}

3
ZRCola/zrcolacomppnl.h
View File

@ -63,7 +63,8 @@ protected:
m_destinationRestyled; ///< Boolean flag to mark destination text is being restyled
wxTextAttr
m_styleNormal, ///< Normal text style
m_stylePUA; ///< PUA character text style
m_stylePUA, ///< PUA character text style
m_styleZRColaUnicodeComposedIssues; ///< ZRCola Unicode Composed issues character text style
std::vector<ZRCola::mapping_vector> m_mapping; ///< Character index mapping vector between source and normalized text
std::pair<long, long>
m_selSource, ///< Character index of selected text in source text control

57
ZRColaCompile/dbsource.cpp
View File

@ -242,6 +242,8 @@ ZRCola::DBSource::~DBSource()
m_comTranslation.free();
m_pCharacterGroup1.free();
m_comCharacterGroup.free();
m_pHighlight1.free();
m_comHighlight.free();
if (m_db)
m_db->Close();
@ -326,6 +328,23 @@ bool ZRCola::DBSource::Open(LPCTSTR filename)
wxVERIFY(SUCCEEDED(params->Append(m_pTranslationSets1)));
}
wxASSERT_MSG(!m_comHighlight, wxT("ADO command already created"));
wxVERIFY(SUCCEEDED(::CoCreateInstance(CLSID_CADOCommand, NULL, CLSCTX_ALL, IID_IADOCommand, (LPVOID*)&m_comHighlight)));
wxVERIFY(SUCCEEDED(m_comHighlight->put_ActiveConnection(variant(m_db))));
wxVERIFY(SUCCEEDED(m_comHighlight->put_CommandType(adCmdText)));
wxVERIFY(SUCCEEDED(m_comHighlight->put_CommandText(bstr(L"SELECT [komb] "
L"FROM [VRS_HighlightChars2] "
L"WHERE [group]=? "
L"ORDER BY [komb]"))));
{
// Create and add command parameters.
com_obj<ADOParameters> params;
wxVERIFY(SUCCEEDED(m_comHighlight->get_Parameters(&params)));
wxASSERT_MSG(!m_pHighlight1, wxT("ADO command parameter already created"));
wxVERIFY(SUCCEEDED(m_comHighlight->CreateParameter(bstr(L"@group"), adSmallInt, adParamInput, 0, variant(DISP_E_PARAMNOTFOUND, VT_ERROR), &m_pHighlight1)));
wxVERIFY(SUCCEEDED(params->Append(m_pHighlight1)));
}
return true;
} else {
_ftprintf(stderr, wxT("%s: error ZCC0011: Could not open database (0x%x).\n"), (LPCTSTR)filename, hr);
@ -1439,3 +1458,41 @@ bool ZRCola::DBSource::GetTagName(const winstd::com_obj<ADORecordset>& rs, tagna
return true;
}
bool ZRCola::DBSource::SelectHighlights(short set, winstd::com_obj<ADORecordset>& rs) const
{
// Create a new recordset.
rs.free();
wxVERIFY(SUCCEEDED(::CoCreateInstance(CLSID_CADORecordset, NULL, CLSCTX_ALL, IID_IADORecordset, (LPVOID*)&rs)));
wxVERIFY(SUCCEEDED(rs->put_CursorLocation(adUseClient)));
wxVERIFY(SUCCEEDED(rs->put_CursorType(adOpenForwardOnly)));
wxVERIFY(SUCCEEDED(rs->put_LockType(adLockReadOnly)));
// Open it.
wxVERIFY(SUCCEEDED(m_pHighlight1->put_Value(variant(set))));
if (FAILED(rs->Open(variant(m_comHighlight), variant(DISP_E_PARAMNOTFOUND, VT_ERROR)))) {
_ftprintf(stderr, wxT("%s: error ZCC0101: Error loading highlights from database. Please make sure the file is ZRCola.zrc compatible.\n"), m_filename.c_str());
LogErrors();
return false;
}
return true;
}
bool ZRCola::DBSource::GetHighlight(const com_obj<ADORecordset>& rs, ZRCola::DBSource::highlight& h) const
{
wxASSERT_MSG(rs, wxT("recordset is empty"));
com_obj<ADOFields> flds;
wxVERIFY(SUCCEEDED(rs->get_Fields(&flds)));
{
com_obj<ADOField> f;
wxVERIFY(SUCCEEDED(flds->get_Item(variant(L"komb"), &f)));
wxCHECK(GetUnicodeString(f, h.chr), false);
}
return true;
}

67
ZRColaCompile/dbsource.h
View File

@ -6,7 +6,7 @@
#pragma once
#include <zrcola/character.h>
#include <zrcola/common.h>
#include <zrcola/highlight.h>
#include <zrcola/language.h>
#include <zrcola/tag.h>
#include <zrcola/translate.h>
@ -404,6 +404,18 @@ namespace ZRCola {
};
///
/// Highlight
///
class highlight {
public:
short set; ///< Highlight set ID
std::wstring chr; ///< Character sequence
inline highlight() : set((short)ZRCOLA_HLGHTSETID_DEFAULT) {}
};
public:
DBSource();
virtual ~DBSource();
@ -452,18 +464,6 @@ namespace ZRCola {
return SUCCEEDED(rs->get_RecordCount(&count)) ? count : (size_t)-1;
}
///
/// Splits string to individual keywords
///
/// \param[in ] str String
/// \param[out] keywords Array of keywords
///
/// \returns
/// - true when successful
/// - false otherwise
///
static bool GetKeywords(const wchar_t *str, std::vector< std::wstring > &keywords);
///
/// Gets boolean from ZRCola.zrc database
///
@ -872,6 +872,30 @@ namespace ZRCola {
///
bool GetTagName(const winstd::com_obj<ADORecordset>& rs, tagname& tn) const;
///
/// Returns character highlights by set
///
/// \param[in ] set Highlight set ID
/// \param[out] rs Recordset with results
///
/// \returns
/// - true when query succeeds
/// - false otherwise
///
bool SelectHighlights(short set, winstd::com_obj<ADORecordset>& rs) const;
///
/// Returns highlight data
///
/// \param[in] rs Recordset with results
/// \param[out] h Highlight
///
/// \returns
/// - true when succeeded
/// - false otherwise
///
bool GetHighlight(const winstd::com_obj<ADORecordset>& rs, highlight& h) const;
protected:
std::basic_string<TCHAR> m_filename; ///< Database filename
winstd::com_obj<ADOConnection> m_db; ///< Database
@ -886,6 +910,9 @@ namespace ZRCola {
winstd::com_obj<ADOCommand> m_comTranslationSets; ///< ADO Command for GetTranslationSeq subquery
winstd::com_obj<ADOParameter> m_pTranslationSets1; ///< \c m_comTranslationSets parameter
winstd::com_obj<ADOCommand> m_comHighlight; ///< ADO Command for SelectHighlights subquery
winstd::com_obj<ADOParameter> m_pHighlight1; ///< \c m_comHighlights parameter
std::set<std::wstring> m_terms_ignore; ///< Terms to ignore when comparing characters
};
};
@ -1098,3 +1125,17 @@ inline ZRCola::tagname_db& operator<<(_Inout_ ZRCola::tagname_db &db, _In_ const
return db;
}
inline ZRCola::highlight_db& operator<<(_Inout_ ZRCola::highlight_db &db, _In_ const ZRCola::DBSource::highlight &rec)
{
unsigned __int32 idx = db.data.size();
db.data.push_back((unsigned __int16)rec.set);
std::wstring::size_type n = rec.chr.length();
wxASSERT_MSG(n <= 0xffff, wxT("character overflow"));
db.data.push_back((unsigned __int16)n);
db.data.insert(db.data.end(), rec.chr.cbegin(), rec.chr.cend());
db.idxChr.push_back(idx);
return db;
}

103
ZRColaCompile/main.cpp
View File

@ -374,9 +374,9 @@ int _tmain(int argc, _TCHAR *argv[])
}
// Preallocate memory.
db_trans.idxSrc.reserve(count);
db_trans.idxDst.reserve(count);
db_trans.data .reserve(count*5);
db_trans.idxSrc.reserve(count*2);
db_trans.idxDst.reserve(count*2);
db_trans.data .reserve(count*2*8);
// Parse translations and build index and data.
ZRCola::DBSource::translation trans;
@ -399,6 +399,7 @@ int _tmain(int argc, _TCHAR *argv[])
}
if (!has_pua) {
trans.set = (short)ZRCOLA_TRANSETID_UNICODE;
trans.dst.rank += 50;
db_trans << trans;
}
}
@ -416,7 +417,13 @@ int _tmain(int argc, _TCHAR *argv[])
{
com_obj<ADORecordset> rs_tran;
if (src.SelectTranslations(static_cast<short>(ZRCOLA_TRANSETID_UNICODE), rs_tran)) {
if (src.GetRecordsetCount(rs_tran) < 0xffffffff) { // 4G check (-1 is reserved for error condition)
size_t count = src.GetRecordsetCount(rs_tran);
if (count < 0xffffffff) { // 4G check (-1 is reserved for error condition)
// Preallocate memory.
db_trans.idxSrc.reserve(db_trans.idxSrc.size() + count);
db_trans.idxDst.reserve(db_trans.idxDst.size() + count);
db_trans.data .reserve(db_trans.data.size() + count*8);
// Parse translations and build temporary database.
ZRCola::DBSource::translation trans;
trans.set = (short)ZRCOLA_TRANSETID_UNICODE;
@ -467,7 +474,13 @@ int _tmain(int argc, _TCHAR *argv[])
// Get translations.
com_obj<ADORecordset> rs_tran;
if (src.SelectTranslations(ts.set, rs_tran)) {
if (src.GetRecordsetCount(rs_tran) < 0xffffffff) { // 4G check (-1 is reserved for error condition)
count = src.GetRecordsetCount(rs_tran);
if (count < 0xffffffff) { // 4G check (-1 is reserved for error condition)
// Preallocate memory.
db_trans.idxSrc.reserve(db_trans.idxSrc.size() + count);
db_trans.idxDst.reserve(db_trans.idxDst.size() + count);
db_trans.data .reserve(db_trans.data.size() + count*8);
// Parse translations and build temporary database.
ZRCola::DBSource::translation trans;
trans.set = ts.set;
@ -500,17 +513,13 @@ int _tmain(int argc, _TCHAR *argv[])
}
}
// Sort indices.
db_transset.idxTranSet.sort();
// Write translation sets to file.
db_transset.idxTranSet.sort();
dst << ZRCola::transet_rec(db_transset);
// Sort indices.
// Write translations to file.
db_trans.idxSrc.sort();
db_trans.idxDst.sort();
// Write translations to file.
dst << ZRCola::translation_rec(db_trans);
{
@ -540,11 +549,9 @@ int _tmain(int argc, _TCHAR *argv[])
has_errors = true;
}
// Sort indices.
// Write translation sequences to file.
db.idxTranSeq.sort();
db.idxRank .sort();
// Write translation sequences to file.
dst << ZRCola::transeq_rec(db);
} else {
_ftprintf(stderr, wxT("%s: error ZCC0025: Error getting translation sequence count from database or too many translation sequences.\n"), (LPCTSTR)filenameIn.c_str());
@ -639,10 +646,8 @@ int _tmain(int argc, _TCHAR *argv[])
has_errors = true;
}
// Sort indices.
db.idxLang.sort();
// Write languages to file.
db.idxLang.sort();
dst << ZRCola::language_rec(db);
} else {
_ftprintf(stderr, wxT("%s: error ZCC0009: Error getting language count from database or too many languages.\n"), (LPCTSTR)filenameIn.c_str());
@ -680,13 +685,11 @@ int _tmain(int argc, _TCHAR *argv[])
has_errors = true;
}
// Sort indices.
// Write language characters to file.
db.idxChr .sort();
#ifdef ZRCOLA_LANGCHAR_LANG_IDX
db.idxLang.sort();
#endif
// Write language characters to file.
dst << ZRCola::langchar_rec(db);
} else {
_ftprintf(stderr, wxT("%s: error ZCC0011: Error getting language characters count from database or too many langchars.\n"), (LPCTSTR)filenameIn.c_str());
@ -729,10 +732,8 @@ int _tmain(int argc, _TCHAR *argv[])
has_errors = true;
}
// Sort indices.
db.idxRank.sort();
// Write character groups to file.
db.idxRank.sort();
dst << ZRCola::chrgrp_rec(db);
} else {
_ftprintf(stderr, wxT("%s: error ZCC0015: Error getting character group count from database or too many character groups.\n"), (LPCTSTR)filenameIn.c_str());
@ -753,7 +754,6 @@ int _tmain(int argc, _TCHAR *argv[])
size_t count = src.GetRecordsetCount(rs);
if (count < 0xffffffff) { // 4G check (-1 is reserved for error condition)
ZRCola::DBSource::character_desc_idx idxChrDsc, idxChrDscSub;
ZRCola::DBSource::character_bank chrs;
// Phase 1: Parse characters and build indexes.
@ -788,14 +788,10 @@ int _tmain(int argc, _TCHAR *argv[])
categories_used.insert(chr->second.cat);
}
// Sort indices.
// Write characters to file.
db.idxChr.sort();
// Save text indices.
idxChrDsc .save(db.idxDsc );
idxChrDscSub.save(db.idxDscSub);
// Write characters to file.
dst << ZRCola::character_rec(db);
} else {
_ftprintf(stderr, wxT("%s: error ZCC0017: Error getting character count from database or too many characters.\n"), (LPCTSTR)filenameIn.c_str());
@ -839,11 +835,9 @@ int _tmain(int argc, _TCHAR *argv[])
has_errors = true;
}
// Sort indices.
// Write character categories to file.
db.idxChrCat.sort();
db.idxRank .sort();
// Write character categories to file.
dst << ZRCola::chrcat_rec(db);
} else {
_ftprintf(stderr, wxT("%s: error ZCC0019: Error getting character category count from database or too many character categories.\n"), (LPCTSTR)filenameIn.c_str());
@ -879,11 +873,9 @@ int _tmain(int argc, _TCHAR *argv[])
has_errors = true;
}
// Sort indices.
// Write characters tags to file.
db.idxChr.sort();
db.idxTag.sort();
// Write characters tags to file.
dst << ZRCola::chrtag_rec(db);
} else {
_ftprintf(stderr, wxT("%s: error ZCC0021: Error getting characters tags count from database or too many character tags.\n"), (LPCTSTR)filenameIn.c_str());
@ -919,11 +911,9 @@ int _tmain(int argc, _TCHAR *argv[])
has_errors = true;
}
// Sort indices.
// Write tags to file.
db.idxName.sort();
db.idxTag .sort();
// Write tags to file.
dst << ZRCola::tagname_rec(db);
} else {
_ftprintf(stderr, wxT("%s: error ZCC0023: Error getting tag name count from database or too many tags.\n"), (LPCTSTR)filenameIn.c_str());
@ -935,6 +925,43 @@ int _tmain(int argc, _TCHAR *argv[])
}
}
{
// Get highlights.
com_obj<ADORecordset> rs;
if (src.SelectHighlights((short)ZRCOLA_HLGHTSETID_ZRCOLA_UNICODE_COMPOSED_ISSUES, rs)) {
size_t count = src.GetRecordsetCount(rs);
if (count < 0xffffffff) { // 4G check (-1 is reserved for error condition)
ZRCola::DBSource::highlight h;
ZRCola::highlight_db db;
// Preallocate memory.
db.idxChr.reserve(count);
db.data .reserve(count*5);
// Parse highlights and build index and data.
h.set = (short)ZRCOLA_HLGHTSETID_ZRCOLA_UNICODE_COMPOSED_ISSUES;
for (; !ZRCola::DBSource::IsEOF(rs); rs->MoveNext()) {
// Read tag name from the database.
if (src.GetHighlight(rs, h)) {
// Add highlight to index and data.
db << h;
} else
has_errors = true;
}
// Write highlights to file.
db.idxChr.sort();
dst << ZRCola::highlight_rec(db);
} else {
_ftprintf(stderr, wxT("%s: error ZCC0027: Error getting highlight count from database or too many tags.\n"), (LPCTSTR)filenameIn.c_str());
has_errors = true;
}
} else {
_ftprintf(stderr, wxT("%s: error ZCC0026: Error getting highlights from database. Please make sure the file is ZRCola.zrc compatible.\n"), (LPCTSTR)filenameIn.c_str());
has_errors = true;
}
}
idrec::close<ZRCola::recordid_t, ZRCola::recordsize_t, ZRCOLA_RECORD_ALIGN>(dst, dst_start);
if (dst.fail()) {

2
lib/libZRCola/build/libZRCola.vcxproj
View File

@ -64,6 +64,7 @@
<ItemGroup>
<ClCompile Include="..\src\character.cpp" />
<ClCompile Include="..\src\common.cpp" />
<ClCompile Include="..\src\highlight.cpp" />
<ClCompile Include="..\src\language.cpp" />
<ClCompile Include="..\src\mapping.cpp" />
<ClCompile Include="..\src\pch.cpp">
@ -75,6 +76,7 @@
<ItemGroup>
<ClInclude Include="..\include\zrcola\character.h" />
<ClInclude Include="..\include\zrcola\common.h" />
<ClInclude Include="..\include\zrcola\highlight.h" />
<ClInclude Include="..\include\zrcola\language.h" />
<ClInclude Include="..\include\zrcola\tag.h" />
<ClInclude Include="..\include\zrcola\translate.h" />

6
lib/libZRCola/build/libZRCola.vcxproj.filters
View File

@ -32,6 +32,9 @@
<ClCompile Include="..\src\tag.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\src\highlight.cpp">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\src\pch.h">
@ -52,5 +55,8 @@
<ClInclude Include="..\include\zrcola\tag.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="..\include\zrcola\highlight.h">
<Filter>Header Files</Filter>
</ClInclude>
</ItemGroup>
</Project>

240
lib/libZRCola/include/zrcola/highlight.h Normal file
View File

@ -0,0 +1,240 @@
/*
SPDX-License-Identifier: GPL-3.0-or-later
Copyright © 2021 Amebis
*/
#pragma once
#include "common.h"
#include <stdex/idrec.h>
#include <functional>
#pragma warning(push)
#pragma warning(disable: 4200)
///
/// Default highlight
///
#define ZRCOLA_HLGHTSETID_DEFAULT ((ZRCola::hlghtsetid_t)0x0000)
///
/// ZRCola Unicode Composed Issues
///
#define ZRCOLA_HLGHTSETID_ZRCOLA_UNICODE_COMPOSED_ISSUES ((ZRCola::hlghtsetid_t)0x0001)
namespace ZRCola {
///
/// Highlight set ID
///
typedef unsigned __int16 hlghtsetid_t;
///
/// Highlight database
///
class highlight_db {
public:
#pragma pack(push)
#pragma pack(2)
///
/// Highlight data
///
struct highlight {
public:
hlghtsetid_t set; ///< Highlight set ID
protected:
unsigned __int16 chr_to; ///< Character end in \c data
wchar_t data[]; ///< Character
private:
inline highlight(_In_ const highlight &other);
inline highlight& operator=(_In_ const highlight &other);
public:
///
/// Constructs the highlight
///
/// \param[in] set Highlight set ID
/// \param[in] chr Character
/// \param[in] chr_len Number of UTF-16 characters in \p chr
///
inline highlight(
_In_opt_ hlghtsetid_t set = 0,
_In_opt_z_count_(chr_len) const wchar_t *chr = NULL,
_In_opt_ size_t chr_len = 0)
{
this->set = set;
this->chr_to = static_cast<unsigned __int16>(chr_len);
if (chr && chr_len) memcpy(this->data, chr, sizeof(wchar_t)*chr_len);
}
inline const wchar_t* chr () const { return data; };
inline wchar_t* chr () { return data; };
inline const wchar_t* chr_end() const { return data + chr_to; };
inline wchar_t* chr_end() { return data + chr_to; };
inline unsigned __int16 chr_len() const { return chr_to; };
inline wchar_t chr_at(_In_ size_t i) const
{
return i < chr_to ? data[i] : 0;
}
};
#pragma pack(pop)
///
/// Highlight index
///
class indexChr : public index<unsigned __int16, unsigned __int32, highlight>
{
public:
///
/// Constructs the index
///
/// \param[in] h Reference to vector holding the data
///
indexChr(_In_ std::vector<unsigned __int16> &h) : index<unsigned __int16, unsigned __int32, highlight>(h) {}
///
/// Compares two highlights by string (for searching)
///
/// \param[in] a Pointer to first element
/// \param[in] b Pointer to second element
///
/// \returns
/// - <0 when a < b
/// - =0 when a == b
/// - >0 when a > b
///
virtual int compare(_In_ const highlight &a, _In_ const highlight &b) const
{
int r = ZRCola::CompareString(a.chr(), a.chr_len(), b.chr(), b.chr_len());
if (r != 0) return r;
if (a.set < b.set) return -1;
else if (a.set > b.set) return +1;
return 0;
}
///
/// Compares two highlights by string (for sorting)
///
/// \param[in] a Pointer to first element
/// \param[in] b Pointer to second element
///
/// \returns
/// - <0 when a < b
/// - =0 when a == b
/// - >0 when a > b
///
virtual int compare_sort(_In_ const highlight &a, _In_ const highlight &b) const
{
// Revert to `compare()` by default.
return compare(a, b);
}
} idxChr; ///< Highlight index
std::vector<unsigned __int16> data; ///< Highlight data
public:
///
/// Constructs the database
///
inline highlight_db() : idxChr(data) {}
///
/// Clears the database
///
inline void clear()
{
idxChr.clear();
data .clear();
}
///
/// Highlights string
///
/// \param[in] input Input string (UTF-16)
/// \param[in] inputMax Length of the input string in characters. Can be (size_t)-1 if \p input is zero terminated.
/// \param[in] callback Function to be called on highlight switch
///
void Highlight(_In_z_count_(inputMax) const wchar_t* input, _In_ size_t inputMax, _In_ std::function<void (hlghtsetid_t set, size_t start, size_t end)> callback) const;
};
typedef stdex::idrec::record<highlight_db, recordid_t, recordsize_t, ZRCOLA_RECORD_ALIGN> highlight_rec;
};
const ZRCola::recordid_t ZRCola::highlight_rec::id = *(ZRCola::recordid_t*)"HGH";
///
/// Writes highlight database to a stream
///
/// \param[in] stream Output stream
/// \param[in] db Highlight database
///
/// \returns The stream \p stream
///
inline std::ostream& operator <<(_In_ std::ostream& stream, _In_ const ZRCola::highlight_db &db)
{
// Write highlight index.
if (stream.fail()) return stream;
stream << db.idxChr;
// Write data count.
auto data_count = db.data.size();
#if defined(_WIN64) || defined(__x86_64__) || defined(__ppc64__)
// 4G check
if (data_count > 0xffffffff) {
stream.setstate(std::ios_base::failbit);
return stream;
}
#endif
if (stream.fail()) return stream;
unsigned __int32 count = (unsigned __int32)data_count;
stream.write((const char*)&count, sizeof(count));
// Write data.
if (stream.fail()) return stream;
stream.write((const char*)db.data.data(), sizeof(unsigned __int16)*static_cast<std::streamsize>(count));
return stream;
}
///
/// Reads highlight database from a stream
///
/// \param[in ] stream Input stream
/// \param[out] db Highlight database
///
/// \returns The stream \p stream
///
inline std::istream& operator >>(_In_ std::istream& stream, _Out_ ZRCola::highlight_db &db)
{
// Read highlight index.
stream >> db.idxChr;
if (!stream.good()) return stream;
// Read data count.
unsigned __int32 count;
stream.read((char*)&count, sizeof(count));
if (!stream.good()) return stream;
if (count) {
// Read data.
db.data.resize(count);
stream.read((char*)db.data.data(), sizeof(unsigned __int16)*static_cast<std::streamsize>(count));
} else
db.data.clear();
return stream;
}
#pragma warning(pop)

77
lib/libZRCola/src/highlight.cpp Normal file
View File

@ -0,0 +1,77 @@
/*
SPDX-License-Identifier: GPL-3.0-or-later
Copyright © 2021 Amebis
*/
#include "pch.h"
_Use_decl_annotations_
void ZRCola::highlight_db::Highlight(const wchar_t* input, size_t inputMax, std::function<void (hlghtsetid_t set, size_t start, size_t end)> callback) const
{
size_t start = 0;
hlghtsetid_t set = ZRCOLA_HLGHTSETID_DEFAULT;
for (size_t i = 0; i < inputMax;) {
// Find the longest matching highlight at i-th character.
size_t l_match = (size_t)-1;
for (size_t l = 0, r = idxChr.size(), ii = i, j = 0; ii < inputMax && l < r; ii++, j++) {
wchar_t c = input[ii];
while (l < r) {
// Test the highlight in the middle of the search area.
size_t m = (l + r) / 2;
// Get the j-th character of the highlight.
// All highlights that get short on characters are lexically ordered before.
// Thus the j-th character is considered 0.
wchar_t s = idxChr[m].chr_at(j);
// Do the bisection test.
if (c < s) r = m;
else if (s < c) l = m + 1;
else {
// Character found.
// Narrow the search area on the left to start at the first highlight in the run.
for (size_t r2 = m; l < r2;) {
size_t m2 = (l + r2) / 2;
if (c <= idxChr[m2].chr_at(j)) r2 = m2; else l = m2 + 1;
}
// Narrow the search area on the right to end at the first highlight not in the run.
for (size_t l2 = m + 1; l2 < r;) {
size_t m2 = (l2 + r) / 2;
if (idxChr[m2].chr_at(j) <= c) l2 = m2 + 1; else r = m2;
}
if (j + 1 == idxChr[l].chr_len()) {
// The first highlight of the run was a match (thus far). Save it.
l_match = l;
}
break;
}
}
}
if (l_match < idxChr.size()) {
// The saved highlight was an exact match.
const highlight &hghl = idxChr[l_match];
if (set != hghl.set) {
callback(set, start, i);
start = i;
set = hghl.set;
}
i += hghl.chr_len();
} else {
// The match was not found.
if (set != ZRCOLA_HLGHTSETID_DEFAULT) {
callback(set, start, i);
start = i;
set = ZRCOLA_HLGHTSETID_DEFAULT;
}
i++;
}
}
callback(set, start, inputMax);
}

1
lib/libZRCola/src/pch.h
View File

@ -8,6 +8,7 @@
#include "../../../include/version.h"
#include "../include/zrcola/character.h"
#include "../include/zrcola/highlight.h"
#include "../include/zrcola/language.h"
#include "../include/zrcola/translate.h"
#include "../include/zrcola/tag.h"

BIN
output/data/ZRCola.zrcdb
View File

Binary file not shown.