Languages and language-specific characters added to the database

ZRColaCompile/dbsource.cpp

@@ -127,6 +127,21 @@ bool ZRCola::DBSource::GetValue(const ATL::CComPtr<ADOField>& f, int& val) const
 }
 
 
+bool ZRCola::DBSource::GetValue(const ATL::CComPtr<ADOField>& f, std::wstring& val) const
+{
+    wxASSERT_MSG(f, wxT("field is empty"));
+
+    ATL::CComVariant v;
+    wxVERIFY(SUCCEEDED(f->get_Value(&v)));
+    wxCHECK(SUCCEEDED(v.ChangeType(VT_BSTR)), false);
+
+    val.reserve(::SysStringLen(V_BSTR(&v)));
+    val = V_BSTR(&v);
+
+    return true;
+}
+
+
 bool ZRCola::DBSource::GetUnicodeCharacter(const ATL::CComPtr<ADOField>& f, wchar_t& chr) const
 {
     wxASSERT_MSG(f, wxT("field is empty"));
@@ -241,6 +256,45 @@ bool ZRCola::DBSource::GetKeyCode(const ATL::CComPtr<ADOField>& f, ZRCola::DBSou
 }
 
 
+bool ZRCola::DBSource::GetLanguage(const ATL::CComPtr<ADOField>& f, ZRCola::langid_t& lang) const
+{
+    wxASSERT_MSG(f, wxT("field is empty"));
+
+    ATL::CComVariant v;
+    wxVERIFY(SUCCEEDED(f->get_Value(&v)));
+    wxCHECK(SUCCEEDED(v.ChangeType(VT_BSTR)), false);
+
+    // Convert to lowercase.
+    _wcslwr_l(V_BSTR(&v), m_locale);
+
+    // Parse the field.
+    size_t n = wcsnlen(V_BSTR(&v), ::SysStringLen(V_BSTR(&v)));
+    if (n != 3) {
+            ATL::CComBSTR fieldname; wxVERIFY(SUCCEEDED(f->get_Name(&fieldname)));
+            _ftprintf(stderr, wxT("%s: error ZCC0080: Syntax error in \"%.*ls\" field (\"%.*ls\"). Language ID must be exactly three (3) characters long.\n"), m_filename.c_str(), fieldname.Length(), (BSTR)fieldname, n, V_BSTR(&v));
+            return false;
+    }
+    for (size_t i = 0;; i++) {
+        if (i < sizeof(lang)) {
+            if (i < n) {
+                wchar_t c = V_BSTR(&v)[i];
+                if ((unsigned short)c > 0x7f) {
+                    ATL::CComBSTR fieldname; wxVERIFY(SUCCEEDED(f->get_Name(&fieldname)));
+                    _ftprintf(stderr, wxT("%s: error ZCC0081: Syntax error in \"%.*ls\" field (\"%.*ls\"). Language ID must contain ASCII characters only.\n"), m_filename.c_str(), fieldname.Length(), (BSTR)fieldname, n, V_BSTR(&v));
+                    return false;
+                }
+                lang[i] = (char)c;
+            } else
+                lang[i] = 0;
+        } else
+            break;
+    }
+
+    return true;
+}
+
+
+
 bool ZRCola::DBSource::SelectTranslations(ATL::CComPtr<ADORecordset> &rs) const
 {
     // Create a new recordset.
@@ -347,3 +401,77 @@ bool ZRCola::DBSource::GetKeySequence(const ATL::CComPtr<ADORecordset>& rs, ZRCo
 
     return true;
 }
+
+
+bool ZRCola::DBSource::SelectLanguages(ATL::CComPtr<ADORecordset> &rs) const
+{
+    // Create a new recordset.
+    if (rs) rs.Release();
+    wxCHECK(SUCCEEDED(::CoCreateInstance(CLSID_CADORecordset, NULL, CLSCTX_ALL, IID_IADORecordset, (LPVOID*)&rs)), false);
+
+    // Open it.
+    if (FAILED(rs->Open(ATL::CComVariant(L"SELECT DISTINCT [entCode] FROM [VRS_Jezik]"), ATL::CComVariant(m_db), adOpenStatic, adLockReadOnly, adCmdText))) {
+        _ftprintf(stderr, wxT("%s: error ZCC0060: Error loading languages from database. Please make sure the file is ZRCola.zrc compatible.\n"), m_filename.c_str());
+        LogErrors();
+        return false;
+    }
+
+    return true;
+}
+
+
+bool ZRCola::DBSource::GetLanguage(const ATL::CComPtr<ADORecordset>& rs, ZRCola::DBSource::language& lang) const
+{
+    wxASSERT_MSG(rs, wxT("recordset is empty"));
+
+    ATL::CComPtr<ADOFields> flds;
+    wxVERIFY(SUCCEEDED(rs->get_Fields(&flds)));
+
+    {
+        ATL::CComPtr<ADOField> f;
+        wxVERIFY(SUCCEEDED(flds->get_Item(ATL::CComVariant(L"entCode"), &f)));
+        wxCHECK(GetLanguage(f, lang.id), false);
+    }
+
+    return true;
+}
+
+
+bool ZRCola::DBSource::SelectLanguageCharacters(ATL::CComPtr<ADORecordset> &rs) const
+{
+    // Create a new recordset.
+    if (rs) rs.Release();
+    wxCHECK(SUCCEEDED(::CoCreateInstance(CLSID_CADORecordset, NULL, CLSCTX_ALL, IID_IADORecordset, (LPVOID*)&rs)), false);
+
+    // Open it.
+    if (FAILED(rs->Open(ATL::CComVariant(L"SELECT DISTINCT [znak], [lang] FROM [VRS_CharLocal]"), ATL::CComVariant(m_db), adOpenStatic, adLockReadOnly, adCmdText))) {
+        _ftprintf(stderr, wxT("%s: error ZCC0090: Error loading language characters from database. Please make sure the file is ZRCola.zrc compatible.\n"), m_filename.c_str());
+        LogErrors();
+        return false;
+    }
+
+    return true;
+}
+
+
+bool ZRCola::DBSource::GetLanguageCharacter(const ATL::CComPtr<ADORecordset>& rs, ZRCola::DBSource::langchar& lc) const
+{
+    wxASSERT_MSG(rs, wxT("recordset is empty"));
+
+    ATL::CComPtr<ADOFields> flds;
+    wxVERIFY(SUCCEEDED(rs->get_Fields(&flds)));
+
+    {
+        ATL::CComPtr<ADOField> f;
+        wxVERIFY(SUCCEEDED(flds->get_Item(ATL::CComVariant(L"znak"), &f)));
+        wxCHECK(GetUnicodeCharacter(f, lc.chr), false);
+    }
+
+    {
+        ATL::CComPtr<ADOField> f;
+        wxVERIFY(SUCCEEDED(flds->get_Item(ATL::CComVariant(L"lang"), &f)));
+        wxCHECK(GetLanguage(f, lc.lang), false);
+    }
+
+    return true;
+}

ZRColaCompile/dbsource.h

@@ -19,6 +19,8 @@
 
 #pragma once
 
+#include <zrcola/language.h>
+
 #include <atlbase.h>
 #include <adoint.h>
 #include <string>
@@ -62,6 +64,25 @@ namespace ZRCola {
             std::vector<keycode> seq;   ///< Key sequence
         };
 
+
+        ///
+        /// Language
+        ///
+        class language {
+        public:
+            ZRCola::langid_t id;        ///< Language ID
+        };
+
+
+        ///
+        /// Language Character
+        ///
+        class langchar {
+        public:
+            wchar_t chr;                ///> Character
+            ZRCola::langid_t lang;      ///< Language ID
+        };
+
     public:
         DBSource();
         virtual ~DBSource();
@@ -141,6 +162,19 @@ namespace ZRCola {
         bool GetValue(const ATL::CComPtr<ADOField>& f, int& val) const;
 
 
+        ///
+        /// Gets string from ZRCola.zrc database
+        ///
+        /// \param[in]  f    Data field
+        /// \param[out] val  Output string value
+        ///
+        /// \returns
+        /// - true when successful
+        /// - false otherwise
+        ///
+        bool GetValue(const ATL::CComPtr<ADOField>& f, std::wstring& val) const;
+
+
         ///
         /// Gets encoded Unicode character from ZRCola.zrc database
         ///
@@ -180,6 +214,19 @@ namespace ZRCola {
         bool GetKeyCode(const ATL::CComPtr<ADOField>& f, keyseq::keycode& kc) const;
 
 
+        ///
+        /// Gets language ID from ZRCola.zrc database
+        ///
+        /// \param[in]  f     Data field
+        /// \param[out] lang  Language
+        ///
+        /// \returns
+        /// - true when successful
+        /// - false otherwise
+        ///
+        bool GetLanguage(const ATL::CComPtr<ADOField>& f, langid_t& lang) const;
+
+
         ///
         /// Returns character translations
         ///
@@ -229,6 +276,56 @@ namespace ZRCola {
         ///
         bool GetKeySequence(const ATL::CComPtr<ADORecordset>& rs, keyseq& ks) const;
 
+
+        ///
+        /// Returns languages
+        ///
+        /// \param[out] rs  Recordset with results
+        ///
+        /// \returns
+        /// - true when query succeeds
+        /// - false otherwise
+        ///
+        bool SelectLanguages(ATL::CComPtr<ADORecordset>& rs) const;
+
+
+        ///
+        /// Returns language data
+        ///
+        /// \param[in]  rs    Recordset with results
+        /// \param[out] lang  Language
+        ///
+        /// \returns
+        /// - true when succeeded
+        /// - false otherwise
+        ///
+        bool GetLanguage(const ATL::CComPtr<ADORecordset>& rs, language& lang) const;
+
+
+        ///
+        /// Returns language character
+        ///
+        /// \param[out] rs  Recordset with results
+        ///
+        /// \returns
+        /// - true when query succeeds
+        /// - false otherwise
+        ///
+        bool SelectLanguageCharacters(ATL::CComPtr<ADORecordset>& rs) const;
+
+
+        ///
+        /// Returns language character data
+        ///
+        /// \param[in]  rs    Recordset with results
+        /// \param[out] lang  Language character data
+        ///
+        /// \returns
+        /// - true when succeeded
+        /// - false otherwise
+        ///
+        bool GetLanguageCharacter(const ATL::CComPtr<ADORecordset>& rs, langchar& lc) const;
+
     protected:
         std::basic_string<TCHAR> m_filename;    ///< Database filename
         ATL::CComPtr<ADOConnection> m_db;       ///< Database

ZRColaCompile/main.cpp

@@ -132,6 +132,117 @@ inline std::ostream& operator <<(std::ostream& stream, const ZRCola::keyseq_db &
 }
 
 
+///
+/// Writes language database to a stream
+///
+/// \param[in] stream  Output stream
+/// \param[in] db      Language database
+///
+/// \returns The stream \p stream
+///
+inline std::ostream& operator <<(std::ostream& stream, const ZRCola::language_db &db)
+{
+    unsigned __int32 count;
+
+    // Write index count.
+    ZRCola::language_db::indexLang::size_type lang_count = db.idxLng.size();
+#if defined(_WIN64) || defined(__x86_64__) || defined(__ppc64__)
+    // 4G check
+    if (lang_count > 0xffffffff) {
+        stream.setstate(std::ios_base::failbit);
+        return stream;
+    }
+#endif
+    if (stream.fail()) return stream;
+    count = (unsigned __int32)lang_count;
+    stream.write((const char*)&count, sizeof(count));
+
+    // Write language index.
+    if (stream.fail()) return stream;
+    stream.write((const char*)db.idxLng.data(), sizeof(unsigned __int32)*count);
+
+
+    // Write data count.
+    std::vector<unsigned __int16>::size_type data_count = db.data.size();
+#if defined(_WIN64) || defined(__x86_64__) || defined(__ppc64__)
+    // 4G check
+    if (data_count > 0xffffffff) {
+        stream.setstate(std::ios_base::failbit);
+        return stream;
+    }
+#endif
+    if (stream.fail()) return stream;
+    count = (unsigned __int32)data_count;
+    stream.write((const char*)&count, sizeof(count));
+
+    // Write data.
+    if (stream.fail()) return stream;
+    stream.write((const char*)db.data.data(), sizeof(unsigned __int16)*count);
+
+    return stream;
+}
+
+
+///
+/// Writes language character database to a stream
+///
+/// \param[in] stream  Output stream
+/// \param[in] db      Language character database
+///
+/// \returns The stream \p stream
+///
+inline std::ostream& operator <<(std::ostream& stream, const ZRCola::langchar_db &db)
+{
+#ifdef ZRCOLA_LANGCHAR_LANG_IDX
+    assert(db.idxChr.size() == db.idxLng.size());
+#endif
+
+    unsigned __int32 count;
+
+    // Write index count.
+    ZRCola::langchar_db::indexChar::size_type lc_count = db.idxChr.size();
+#if defined(_WIN64) || defined(__x86_64__) || defined(__ppc64__)
+    // 4G check
+    if (lc_count > 0xffffffff) {
+        stream.setstate(std::ios_base::failbit);
+        return stream;
+    }
+#endif
+    if (stream.fail()) return stream;
+    count = (unsigned __int32)lc_count;
+    stream.write((const char*)&count, sizeof(count));
+
+    // Write character index.
+    if (stream.fail()) return stream;
+    stream.write((const char*)db.idxChr.data(), sizeof(unsigned __int32)*count);
+
+#ifdef ZRCOLA_LANGCHAR_LANG_IDX
+    // Write language index.
+    if (stream.fail()) return stream;
+    stream.write((const char*)db.idxLng.data(), sizeof(unsigned __int32)*count);
+#endif
+
+    // Write data count.
+    std::vector<unsigned __int16>::size_type data_count = db.data.size();
+#if defined(_WIN64) || defined(__x86_64__) || defined(__ppc64__)
+    // 4G check
+    if (data_count > 0xffffffff) {
+        stream.setstate(std::ios_base::failbit);
+        return stream;
+    }
+#endif
+    if (stream.fail()) return stream;
+    count = (unsigned __int32)data_count;
+    stream.write((const char*)&count, sizeof(count));
+
+    // Write data.
+    if (stream.fail()) return stream;
+    stream.write((const char*)db.data.data(), sizeof(unsigned __int16)*count);
+
+    return stream;
+}
+
+
 ///
 /// Main function
 ///
@@ -259,7 +370,6 @@ int _tmain(int argc, _TCHAR *argv[])
         }
     }
 
-
     {
         // Get key sequences.
         ATL::CComPtr<ADORecordset> rs;
@@ -329,6 +439,102 @@ int _tmain(int argc, _TCHAR *argv[])
         }
     }
 
+    {
+        // Get languages.
+        ATL::CComPtr<ADORecordset> rs;
+        if (src.SelectLanguages(rs)) {
+            size_t count = src.GetRecordsetCount(rs);
+            if (count < 0xffffffff) { // 4G check (-1 is reserved for error condition)
+                ZRCola::DBSource::language lang;
+                ZRCola::language_db db;
+
+                // Preallocate memory.
+                db.idxLng.reserve(count);
+                db.data  .reserve(count*4);
+
+                // Parse languages and build index and data.
+                while (!ZRCola::DBSource::IsEOF(rs)) {
+                    // Read language from the database.
+                    if (src.GetLanguage(rs, lang)) {
+                        // Add language to index and data.
+                        unsigned __int32 idx = db.data.size();
+                        for (std::wstring::size_type i = 0; i < sizeof(ZRCola::langid_t)/sizeof(unsigned __int16); i++)
+                            db.data.push_back(((const unsigned __int16*)lang.id)[i]);
+                        db.idxLng.push_back(idx);
+                    } else
+                        has_errors = true;
+
+                    wxVERIFY(SUCCEEDED(rs->MoveNext()));
+                }
+
+                // Sort indices.
+                db.idxLng.sort();
+
+                // Write languages to file.
+                dst << ZRCola::language_rec(db);
+            } else {
+                _ftprintf(stderr, wxT("%s: error ZCC0009: Error getting language count from database or too many languages.\n"), (LPCTSTR)filenameIn.c_str());
+                has_errors = true;
+            }
+        } else {
+            _ftprintf(stderr, wxT("%s: error ZCC0008: Error getting languages from database. Please make sure the file is ZRCola.zrc compatible.\n"), (LPCTSTR)filenameIn.c_str());
+            has_errors = true;
+        }
+    }
+
+    {
+        // Get language characters.
+        ATL::CComPtr<ADORecordset> rs;
+        if (src.SelectLanguageCharacters(rs)) {
+            size_t count = src.GetRecordsetCount(rs);
+            if (count < 0xffffffff) { // 4G check (-1 is reserved for error condition)
+                ZRCola::DBSource::langchar lc;
+                ZRCola::langchar_db db;
+
+                // Preallocate memory.
+                db.idxChr.reserve(count);
+#ifdef ZRCOLA_LANGCHAR_LANG_IDX
+                db.idxLng.reserve(count);
+#endif
+                db.data  .reserve(count*4);
+
+                // Parse language characters and build index and data.
+                while (!ZRCola::DBSource::IsEOF(rs)) {
+                    // Read language characters from the database.
+                    if (src.GetLanguageCharacter(rs, lc)) {
+                        // Add language characters to index and data.
+                        unsigned __int32 idx = db.data.size();
+                        db.data.push_back(lc.chr);
+                        for (std::wstring::size_type i = 0; i < sizeof(ZRCola::langid_t)/sizeof(unsigned __int16); i++)
+                            db.data.push_back(((const unsigned __int16*)lc.lang)[i]);
+                        db.idxChr.push_back(idx);
+#ifdef ZRCOLA_LANGCHAR_LANG_IDX
+                        db.idxLng.push_back(idx);
+#endif
+                    } else
+                        has_errors = true;
+
+                    wxVERIFY(SUCCEEDED(rs->MoveNext()));
+                }
+
+                // Sort indices.
+                db.idxChr .sort();
+#ifdef ZRCOLA_LANGCHAR_LANG_IDX
+                db.idxLng.sort();
+#endif
+
+                // Write language characters to file.
+                dst << ZRCola::langchar_rec(db);
+            } else {
+                _ftprintf(stderr, wxT("%s: error ZCC0011: Error getting language characters count from database or too many langchars.\n"), (LPCTSTR)filenameIn.c_str());
+                has_errors = true;
+            }
+        } else {
+            _ftprintf(stderr, wxT("%s: error ZCC0010: Error getting language characters from database. Please make sure the file is ZRCola.zrc compatible.\n"), (LPCTSTR)filenameIn.c_str());
+            has_errors = true;
+        }
+    }
+
     stdex::idrec::close<ZRCola::recordid_t, ZRCola::recordsize_t, ZRCOLA_RECORD_ALIGN>(dst, dst_start);
 
     if (dst.fail()) {

ZRColaCompile/stdafx.h

@@ -24,6 +24,7 @@
 #include "../include/zrcola.h"
 #include "dbsource.h"
 
+#include <zrcola/language.h>
 #include <zrcola/translate.h>
 #include <zrcolaui/keyboard.h>
 

lib/libZRCola/build/libZRCola.vcxproj

@@ -19,6 +19,7 @@
     </ProjectConfiguration>
   </ItemGroup>
   <ItemGroup>
+    <ClCompile Include="..\src\language.cpp" />
     <ClCompile Include="..\src\mapping.cpp" />
     <ClCompile Include="..\src\normalize.cpp" />
     <ClCompile Include="..\src\stdafx.cpp">
@@ -31,6 +32,7 @@
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\include\zrcola\common.h" />
+    <ClInclude Include="..\include\zrcola\language.h" />
     <ClInclude Include="..\include\zrcola\normalize.h" />
     <ClInclude Include="..\include\zrcola\translate.h" />
     <ClInclude Include="..\src\stdafx.h" />

lib/libZRCola/build/libZRCola.vcxproj.filters

@@ -27,6 +27,9 @@
     <ClCompile Include="..\src\translate.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\src\language.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\src\stdafx.h">
@@ -41,6 +44,9 @@
     <ClInclude Include="..\include\zrcola\translate.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="..\include\zrcola\language.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ResourceCompile Include="..\res\libZRCola.rc">

lib/libZRCola/include/zrcola/language.h

@@ -0,0 +1,358 @@
+/*
+    Copyright 2015-2016 Amebis
+
+    This file is part of ZRCola.
+
+    ZRCola is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    ZRCola is distributed in the hope that it will be useful, but
+    WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with ZRCola. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include "common.h"
+
+#include <stdex/idrec.h>
+#include <istream>
+#include <vector>
+#include <string>
+
+#pragma warning(push)
+#pragma warning(disable: 4200)
+#pragma warning(disable: 4251)
+#pragma warning(disable: 4512)
+
+
+namespace ZRCola {
+    ///
+    /// Language ID type
+    /// Three letter abbreviation, zero terminated
+    ///
+    typedef char langid_t[4];
+
+    ///
+    /// Language Character Database
+    ///
+    class ZRCOLA_API langchar_db {
+    public:
+#pragma pack(push)
+#pragma pack(2)
+        ///
+        /// Character data
+        ///
+        struct langchar {
+            wchar_t chr;                ///> Character
+            langid_t lang;              ///< Language ID
+        };
+#pragma pack(pop)
+
+        ///
+        /// Character index
+        ///
+        class indexChar : public index<unsigned __int16, unsigned __int32, langchar>
+        {
+        public:
+            ///
+            /// Constructs the index
+            ///
+            /// \param[in] h  Reference to vector holding the data
+            ///
+            indexChar(_In_ std::vector<unsigned __int16> &h) : index<unsigned __int16, unsigned __int32, langchar>(h) {}
+
+            ///
+            /// Compares two characters by ID (for searching)
+            ///
+            /// \param[in] a  Pointer to first element
+            /// \param[in] b  Pointer to second element
+            ///
+            /// \returns
+            /// - <0 when a <  b
+            /// - =0 when a == b
+            /// - >0 when a >  b
+            ///
+            virtual int compare(_In_ const langchar &a, _In_ const langchar &b) const
+            {
+                     if (a.chr < b.chr) return -1;
+                else if (a.chr > b.chr) return  1;
+
+                return 0;
+            }
+
+            ///
+            /// Compares two characters by ID (for sorting)
+            ///
+            /// \param[in] a  Pointer to first element
+            /// \param[in] b  Pointer to second element
+            ///
+            /// \returns
+            /// - <0 when a <  b
+            /// - =0 when a == b
+            /// - >0 when a >  b
+            ///
+            virtual int compare_sort(_In_ const langchar &a, _In_ const langchar &b) const
+            {
+                     if (a.chr < b.chr) return -1;
+                else if (a.chr > b.chr) return  1;
+
+                int r = memcmp(a.lang, b.lang, sizeof(langid_t));
+                if (r != 0) return r;
+
+                return 0;
+            }
+        } idxChr;      ///< Character index
+
+
+#ifdef ZRCOLA_LANGCHAR_LANG_IDX
+        ///
+        /// Character Language Index
+        ///
+        class indexCharLang : public index<unsigned __int16, unsigned __int32, langchar>
+        {
+        public:
+            ///
+            /// Constructs the index
+            ///
+            /// \param[in] h  Reference to vector holding the data
+            ///
+            indexCharLang(_In_ std::vector<unsigned __int16> &h) : index<unsigned __int16, unsigned __int32, langchar>(h) {}
+
+            ///
+            /// Compares two languages by ID (for searching)
+            ///
+            /// \param[in] a  Pointer to first element
+            /// \param[in] b  Pointer to second element
+            ///
+            /// \returns
+            /// - <0 when a <  b
+            /// - =0 when a == b
+            /// - >0 when a >  b
+            ///
+            virtual int compare(_In_ const langchar &a, _In_ const langchar &b) const
+            {
+                int r = memcmp(a.lang, b.lang, sizeof(langid_t));
+                if (r != 0) return r;
+
+                return 0;
+            }
+
+            ///
+            /// Compares two languages by ID (for sorting)
+            ///
+            /// \param[in] a  Pointer to first element
+            /// \param[in] b  Pointer to second element
+            ///
+            /// \returns
+            /// - <0 when a <  b
+            /// - =0 when a == b
+            /// - >0 when a >  b
+            ///
+            virtual int compare_sort(_In_ const langchar &a, _In_ const langchar &b) const
+            {
+                int r = memcmp(a.lang, b.lang, sizeof(langid_t));
+                if (r != 0) return r;
+
+                     if (a.chr < b.chr) return -1;
+                else if (a.chr > b.chr) return  1;
+
+                return 0;
+            }
+        } idxLng;      ///< Character language index
+#endif
+
+        std::vector<unsigned __int16> data;     ///< Character data
+
+    public:
+        ///
+        /// Constructs the database
+        ///
+        inline langchar_db() : idxChr(data)
+#ifdef ZRCOLA_LANGCHAR_LANG_IDX
+            , idxLng(data)
+#endif
+        {}
+
+        ///
+        /// Tests presence of character in the given language
+        ///
+        /// \param[in]  chr   Character (UTF-16)
+        /// \param[in]  lang  Language
+        ///
+        /// \returns
+        /// - \c true when character is used in language
+        /// - \c false otherwise
+        bool IsLocalCharacter(_In_ wchar_t chr, _In_ langid_t lang) const;
+    };
+
+
+    typedef ZRCOLA_API stdex::idrec::record<langchar_db, recordid_t, recordsize_t, ZRCOLA_RECORD_ALIGN> langchar_rec;
+
+
+    ///
+    /// Language database
+    ///
+    class ZRCOLA_API language_db {
+    public:
+#pragma pack(push)
+#pragma pack(2)
+        ///
+        /// Language data
+        ///
+        struct language {
+            langid_t id;                ///< Language ID
+        };
+#pragma pack(pop)
+
+        ///
+        /// Language index
+        ///
+        class indexLang : public index<unsigned __int16, unsigned __int32, language>
+        {
+        public:
+            ///
+            /// Constructs the index
+            ///
+            /// \param[in] h  Reference to vector holding the data
+            ///
+            indexLang(_In_ std::vector<unsigned __int16> &h) : index<unsigned __int16, unsigned __int32, language>(h) {}
+
+            ///
+            /// Compares two languages by ID (for searching)
+            ///
+            /// \param[in] a  Pointer to first element
+            /// \param[in] b  Pointer to second element
+            ///
+            /// \returns
+            /// - <0 when a <  b
+            /// - =0 when a == b
+            /// - >0 when a >  b
+            ///
+            virtual int compare(_In_ const language &a, _In_ const language &b) const
+            {
+                int r = memcmp(a.id, b.id, sizeof(langid_t));
+                if (r != 0) return r;
+
+                return 0;
+            }
+
+            ///
+            /// Compares two languages by ID (for sorting)
+            ///
+            /// \param[in] a  Pointer to first element
+            /// \param[in] b  Pointer to second element
+            ///
+            /// \returns
+            /// - <0 when a <  b
+            /// - =0 when a == b
+            /// - >0 when a >  b
+            ///
+            virtual int compare_sort(_In_ const language &a, _In_ const language &b) const
+            {
+                int r = memcmp(a.id, b.id, sizeof(langid_t));
+                if (r != 0) return r;
+
+                // As the language ID must not duplicate, further comparison is pointless.
+
+                return 0;
+            }
+        } idxLng;      ///< Language index
+
+        std::vector<unsigned __int16> data;     ///< Language data
+
+    public:
+        ///
+        /// Constructs the database
+        ///
+        inline language_db() : idxLng(data) {}
+    };
+
+
+    typedef ZRCOLA_API stdex::idrec::record<language_db, recordid_t, recordsize_t, ZRCOLA_RECORD_ALIGN> language_rec;
+};
+
+
+const ZRCola::recordid_t stdex::idrec::record<ZRCola::langchar_db, ZRCola::recordid_t, ZRCola::recordsize_t, ZRCOLA_RECORD_ALIGN>::id = *(ZRCola::recordid_t*)"L-C";
+const ZRCola::recordid_t stdex::idrec::record<ZRCola::language_db, ZRCola::recordid_t, ZRCola::recordsize_t, ZRCOLA_RECORD_ALIGN>::id = *(ZRCola::recordid_t*)"LNG";
+
+
+///
+/// Reads language character database from a stream
+///
+/// \param[in]  stream  Input stream
+/// \param[out] db      Language character database
+///
+/// \returns The stream \p stream
+///
+inline std::istream& operator >>(_In_ std::istream& stream, _Out_ ZRCola::langchar_db &db)
+{
+    unsigned __int32 count;
+
+    // Read index count.
+    stream.read((char*)&count, sizeof(count));
+    if (!stream.good()) return stream;
+
+    // Read character index.
+    db.idxChr.resize(count);
+    stream.read((char*)db.idxChr.data(), sizeof(unsigned __int32)*count);
+    if (!stream.good()) return stream;
+
+#ifdef ZRCOLA_LANGCHAR_LANG_IDX
+    // Read language index.
+    db.idxLng.resize(count);
+    stream.read((char*)db.idxLng.data(), sizeof(unsigned __int32)*count);
+    if (!stream.good()) return stream;
+#endif
+
+    // Read data count.
+    stream.read((char*)&count, sizeof(count));
+    if (!stream.good()) return stream;
+
+    // Read data.
+    db.data.resize(count);
+    stream.read((char*)db.data.data(), sizeof(unsigned __int16)*count);
+
+    return stream;
+}
+
+
+///
+/// Reads language database from a stream
+///
+/// \param[in]  stream  Input stream
+/// \param[out] db      Language database
+///
+/// \returns The stream \p stream
+///
+inline std::istream& operator >>(_In_ std::istream& stream, _Out_ ZRCola::language_db &db)
+{
+    unsigned __int32 count;
+
+    // Read index count.
+    stream.read((char*)&count, sizeof(count));
+    if (!stream.good()) return stream;
+
+    // Read language index.
+    db.idxLng.resize(count);
+    stream.read((char*)db.idxLng.data(), sizeof(unsigned __int32)*count);
+    if (!stream.good()) return stream;
+
+    // Read data count.
+    stream.read((char*)&count, sizeof(count));
+    if (!stream.good()) return stream;
+
+    // Read data.
+    db.data.resize(count);
+    stream.read((char*)db.data.data(), sizeof(unsigned __int16)*count);
+
+    return stream;
+}
+
+#pragma warning(pop)

lib/libZRCola/src/language.cpp

@@ -0,0 +1,46 @@
+/*
+    Copyright 2015-2016 Amebis
+
+    This file is part of ZRCola.
+
+    ZRCola is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    ZRCola is distributed in the hope that it will be useful, but
+    WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with ZRCola. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "stdafx.h"
+
+
+bool ZRCola::langchar_db::IsLocalCharacter(_In_ wchar_t chr, _In_ ZRCola::langid_t lang) const
+{
+    for (size_t l = 0, r = idxChr.size(); l < r; ) {
+        // Test the character in the middle of the search area.
+        size_t m = (l + r) / 2;
+        const langchar &lc = idxChr[m];
+
+        // Do the bisection test on character.
+             if (chr    < lc.chr) r = m;
+        else if (lc.chr < chr   ) l = m + 1;
+        else {
+            // Do the bisection test on language.
+            int res = memcmp(lang, lc.lang, sizeof(langid_t));
+                 if (res < 0) r = m;
+            else if (res > 0) l = m + 1;
+            else {
+                // Match found.
+                return true;
+            }
+        }
+    }
+
+    return false;
+}

lib/libZRCola/src/stdafx.h

@@ -20,7 +20,9 @@
 #pragma once
 
 #include "../../../include/zrcola.h"
-#include "../include/zrcola/translate.h"
+
+#include "../include/zrcola/language.h"
 #include "../include/zrcola/normalize.h"
+#include "../include/zrcola/translate.h"
 
 #include <assert.h>