Amebis/ZRCola
1
1
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 49 Wiki Activity

Character Select finished

Browse Source 
(closes #11)
This commit is contained in:
Simon Rozman
2016-05-13 03:44:28 +02:00
parent 53ce3a2411
commit 9c3c1585d5
34 changed files with 3380 additions and 763 deletions
Download Patch File Download Diff File Expand all files Collapse all files

198
lib/libZRCola/include/zrcola/character.h
View File

@@ -23,6 +23,8 @@
#include <stdex/idrec.h>
#include <istream>
#include <map>
#include <ostream>
#include <vector>
#include <string>
@@ -37,7 +39,55 @@ namespace ZRCola {
/// Character category ID type
/// Two letter abbreviation, non-terminated
///
typedef char chrcatid_t[2];
struct chrcatid_t {
char data[2];
inline chrcatid_t& operator=(const chrcatid_t &src)
{
data[0] = src.data[0];
data[1] = src.data[1];
return *this;
}
};
///
/// Compares two character category IDs
///
/// \param[in] a First character category ID
/// \param[in] b Second character category ID
///
/// \returns
/// - true when \p a < \p b
/// - false otherwise
///
inline bool operator<(const chrcatid_t& a, const chrcatid_t& b)
{
if (a.data[0] < b.data[0]) return true;
else if (a.data[0] > b.data[0]) return false;
else if (a.data[1] < b.data[1]) return true;
else return false;
}
///
/// Compares two character category IDs
///
/// \param[in] a First character category ID
/// \param[in] b Second character category ID
///
/// \returns
/// - true when \p a > \p b
/// - false otherwise
///
inline bool operator>(const chrcatid_t& a, const chrcatid_t& b)
{
if (a.data[0] > b.data[0]) return true;
else if (a.data[0] < b.data[0]) return false;
else if (a.data[1] > b.data[1]) return true;
else return false;
}
///
/// Character Database
@@ -91,13 +141,17 @@ namespace ZRCola {
}
} idxChr; ///< Character index
std::vector<unsigned __int16> data; ///< Character data
textindex<wchar_t, wchar_t, unsigned __int32> idxDsc; ///< Description index
textindex<wchar_t, wchar_t, unsigned __int32> idxDscSub; ///< Description index (sub-terms)
std::vector<unsigned __int16> data; ///< Character data
public:
///
/// Constructs the database
///
inline character_db() : idxChr(data) {}
void search_by_desc(_In_z_ const wchar_t *str, _Inout_ std::map<wchar_t, unsigned long> &hits, _Inout_ std::map<wchar_t, unsigned long> &hits_sub) const;
};
@@ -148,10 +202,9 @@ namespace ZRCola {
///
virtual int compare(_In_ const chrcat &a, _In_ const chrcat &b) const
{
int r = memcmp(a.id, b.id, sizeof(chrcatid_t));
if (r != 0) return r;
return 0;
if (a.id < b.id) return -1;
else if (a.id < b.id) return 1;
else return 0;
}
} idxChrCat; ///< Character category index
@@ -233,31 +286,118 @@ const ZRCola::recordid_t stdex::idrec::record<ZRCola::chrcat_db, ZRCola::recordi
///
/// Reads character database from a stream
///
/// \param[in] stream Input stream
/// \param[in ] stream Input stream
/// \param[out] db Character database
///
/// \returns The stream \p stream
///
inline std::istream& operator >>(_In_ std::istream& stream, _Out_ ZRCola::character_db &db)
{
unsigned __int32 count;
// Read index count.
stream.read((char*)&count, sizeof(count));
// Read character index.
stream >> db.idxChr;
if (!stream.good()) return stream;
// Read character index.
db.idxChr.resize(count);
stream.read((char*)db.idxChr.data(), sizeof(unsigned __int32)*count);
// Read description index.
stream >> db.idxDsc;
if (!stream.good()) return stream;
// Read sub-term description index.
stream >> db.idxDscSub;
if (!stream.good()) return stream;
// Read data count.
unsigned __int32 count;
stream.read((char*)&count, sizeof(count));
if (!stream.good()) return stream;
// Read data.
db.data.resize(count);
stream.read((char*)db.data.data(), sizeof(unsigned __int16)*count);
if (count) {
// Read data.
db.data.resize(count);
stream.read((char*)db.data.data(), sizeof(unsigned __int16)*count);
} else
db.data.clear();
return stream;
}
///
/// Writes character database to a stream
///
/// \param[in] stream Output stream
/// \param[in] db Character database
///
/// \returns The stream \p stream
///
inline std::ostream& operator <<(_In_ std::ostream& stream, _In_ const ZRCola::character_db &db)
{
// Write character index.
if (stream.fail()) return stream;
stream << db.idxChr;
// Write description index.
if (!stream.good()) return stream;
stream << db.idxDsc;
// Write sub-term description index.
if (!stream.good()) return stream;
stream << db.idxDscSub;
// Write data count.
std::vector<unsigned __int16>::size_type data_count = db.data.size();
#if defined(_WIN64) || defined(__x86_64__) || defined(__ppc64__)
// 4G check
if (data_count > 0xffffffff) {
stream.setstate(std::ios_base::failbit);
return stream;
}
#endif
if (stream.fail()) return stream;
unsigned __int32 count = (unsigned __int32)data_count;
stream.write((const char*)&count, sizeof(count));
// Write data.
if (stream.fail()) return stream;
stream.write((const char*)db.data.data(), sizeof(unsigned __int16)*count);
return stream;
}
///
/// Writes character category database to a stream
///
/// \param[in] stream Output stream
/// \param[in] db Character category database
///
/// \returns The stream \p stream
///
inline std::ostream& operator <<(_In_ std::ostream& stream, _In_ const ZRCola::chrcat_db &db)
{
// Write character category index.
if (stream.fail()) return stream;
stream << db.idxChrCat;
// Write rank index.
if (stream.fail()) return stream;
stream << db.idxRnk;
// Write data count.
std::vector<unsigned __int16>::size_type data_count = db.data.size();
#if defined(_WIN64) || defined(__x86_64__) || defined(__ppc64__)
// 4G check
if (data_count > 0xffffffff) {
stream.setstate(std::ios_base::failbit);
return stream;
}
#endif
if (stream.fail()) return stream;
unsigned __int32 count = (unsigned __int32)data_count;
stream.write((const char*)&count, sizeof(count));
// Write data.
if (stream.fail()) return stream;
stream.write((const char*)db.data.data(), sizeof(unsigned __int16)*count);
return stream;
}
@@ -266,36 +406,32 @@ inline std::istream& operator >>(_In_ std::istream& stream, _Out_ ZRCola::charac
///
/// Reads character category database from a stream
///
/// \param[in] stream Input stream
/// \param[in ] stream Input stream
/// \param[out] db Character category database
///
/// \returns The stream \p stream
///
inline std::istream& operator >>(_In_ std::istream& stream, _Out_ ZRCola::chrcat_db &db)
{
unsigned __int32 count;
// Read index count.
stream.read((char*)&count, sizeof(count));
if (!stream.good()) return stream;
// Read character category index.
db.idxChrCat.resize(count);
stream.read((char*)db.idxChrCat.data(), sizeof(unsigned __int32)*count);
stream >> db.idxChrCat;
if (!stream.good()) return stream;
// Read rank index.
db.idxRnk.resize(count);
stream.read((char*)db.idxRnk.data(), sizeof(unsigned __int32)*count);
stream >> db.idxRnk;
if (!stream.good()) return stream;
// Read data count.
unsigned __int32 count;
stream.read((char*)&count, sizeof(count));
if (!stream.good()) return stream;
// Read data.
db.data.resize(count);
stream.read((char*)db.data.data(), sizeof(unsigned __int16)*count);
if (count) {
// Read data.
db.data.resize(count);
stream.read((char*)db.data.data(), sizeof(unsigned __int16)*count);
} else
db.data.clear();
return stream;
}

268
lib/libZRCola/include/zrcola/common.h
View File

@@ -19,6 +19,9 @@
#pragma once
#include <istream>
#include <ostream>
#include <utility>
#include <vector>
#ifdef _WIN32
#include <Windows.h>
@@ -60,6 +63,20 @@ namespace ZRCola {
typedef unsigned __int32 recordsize_t;
///
/// Key-value index pair for mappings
///
#pragma pack(push)
#pragma pack(2)
template <class T>
struct mappair_t
{
T idx_key; ///< Index of key
T idx_val; ///< Index of value
};
#pragma pack(pop)
///
/// Language ID type
/// Three letter abbreviation, zero terminated
@@ -207,7 +224,7 @@ namespace ZRCola {
for (start = 0, end = size(); start < end; ) {
size_type m = (start + end) / 2;
int r = compare(el, at(m));
if (r < 0) end = m;
if (r < 0) end = m;
else if (r > 0) start = m + 1;
else {
// Narrow the search area on the left to start at the first element in the run.
@@ -240,6 +257,75 @@ namespace ZRCola {
};
///
/// Memory text index
///
template <class T_key, class T_val, class T_idx = unsigned __int32>
class textindex : public std::vector< mappair_t<T_idx> >
{
public:
typedef std::vector< mappair_t<T_idx> > base_t;
std::vector<T_key> keys; ///< Key data
std::vector<T_val> values; ///< Index values
public:
///
/// Constructs the index
///
textindex() {}
///
/// Finds data for given key
///
/// \param[in ] key Pointer to key
/// \param[in ] key_len Count of \p key elements
/// \param[out] val Pointer to receive pointer to key's values
/// \param[out] val_len Pointer to receive count of \p val elements
///
/// \returns
/// - \c true if found
/// - \c false otherwise
///
bool find(_In_count_(key_len) const T_key *key, _In_ size_t key_len, _Out_ const T_val **val, _Out_ size_t *val_len) const
{
for (size_type start = 0, end = size(); start < end; ) {
size_type m = (start + end) / 2;
int r = compare(key, key_len, m);
if (r < 0) end = m;
else if (r > 0) start = m + 1;
else {
// Get values at position m.
size_t start = base_t::at(m ).idx_val;
*val_len = (m < size() ? base_t::at(m + 1).idx_val : values.size()) - start;
*val = &values.at(start);
return true;
}
}
return false;
}
protected:
inline int compare(_In_count_(key_len) const T_key *key, _In_ size_t key_len, size_type pos) const
{
// Get key at position pos.
size_t
start = base_t::at(pos ).idx_key,
key2_len = (pos < size() ? base_t::at(pos + 1).idx_key : keys.size()) - start;
std::vector<T_key>::const_pointer key2 = &keys.at(start);
// Compare keys.
int r = memcmp(key, key2, sizeof(T_key)*std::min<size_t>(key_len, key2_len));
if (r != 0 ) return r;
else if (key_len < key2_len) return -1;
else if (key_len > key2_len) return 1;
return 0;
}
};
///
/// Source-destination index transformation mapping
///
@@ -292,4 +378,184 @@ namespace ZRCola {
};
};
///
/// Writes index to a stream
///
/// \param[in] stream Output stream
/// \param[in] idx Index
///
/// \returns The stream \p stream
///
template <class T, class T_idx, class T_data>
inline std::ostream& operator <<(_In_ std::ostream& stream, _In_ const ZRCola::index<T, T_idx, T_data> &idx)
{
// Write index count.
ZRCola::index<T, T_idx, T_data>::size_type idx_count = idx.size();
#if defined(_WIN64) || defined(__x86_64__) || defined(__ppc64__)
// 4G check
if (idx_count > 0xffffffff) {
stream.setstate(std::ios_base::failbit);
return stream;
}
#endif
if (stream.fail()) return stream;
unsigned __int32 count = (unsigned __int32)idx_count;
stream.write((const char*)&count, sizeof(count));
// Write index data.
if (stream.fail()) return stream;
stream.write((const char*)idx.data(), sizeof(T_idx)*count);
return stream;
}
///
/// Reads index from a stream
///
/// \param[in] stream Input stream
/// \param[out] idx Index
///
/// \returns The stream \p stream
///
template <class T, class T_idx, class T_data>
inline std::istream& operator >>(_In_ std::istream& stream, _Out_ ZRCola::index<T, T_idx, T_data> &idx)
{
unsigned __int32 count;
// Read index count.
stream.read((char*)&count, sizeof(count));
if (!stream.good()) return stream;
if (count) {
// Read index data.
idx.resize(count);
stream.read((char*)idx.data(), sizeof(T_idx)*count);
} else
idx.clear();
return stream;
}
///
/// Writes text index to a stream
///
/// \param[in] stream Output stream
/// \param[in] idx Text index
///
/// \returns The stream \p stream
///
template <class T_key, class T_val, class T_idx>
inline std::ostream& operator <<(_In_ std::ostream& stream, _In_ const ZRCola::textindex<T_key, T_val, T_idx> &idx)
{
unsigned __int32 count;
// Write index count.
ZRCola::textindex<T_key, T_val, T_idx>::size_type idx_count = idx.size();
#if defined(_WIN64) || defined(__x86_64__) || defined(__ppc64__)
// 4G check
if (idx_count > 0xffffffff) {
stream.setstate(std::ios_base::failbit);
return stream;
}
#endif
if (stream.fail()) return stream;
count = (unsigned __int32)idx_count;
stream.write((const char*)&count, sizeof(count));
// Write index data.
if (stream.fail()) return stream;
stream.write((const char*)idx.data(), sizeof(ZRCola::textindex<T_key, T_val, T_idx>::value_type)*count);
// Write key count.
std::vector<T_key>::size_type key_count = idx.keys.size();
#if defined(_WIN64) || defined(__x86_64__) || defined(__ppc64__)
// 4G check
if (idx_count > 0xffffffff) {
stream.setstate(std::ios_base::failbit);
return stream;
}
#endif
if (stream.fail()) return stream;
count = (unsigned __int32)key_count;
stream.write((const char*)&count, sizeof(count));
// Write key data.
if (stream.fail()) return stream;
stream.write((const char*)idx.keys.data(), sizeof(std::vector<T_key>::value_type)*count);
// Write value count.
std::vector<T_val>::size_type value_count = idx.values.size();
#if defined(_WIN64) || defined(__x86_64__) || defined(__ppc64__)
// 4G check
if (idx_count > 0xffffffff) {
stream.setstate(std::ios_base::failbit);
return stream;
}
#endif
if (stream.fail()) return stream;
count = (unsigned __int32)value_count;
stream.write((const char*)&count, sizeof(count));
// Write value data.
if (stream.fail()) return stream;
stream.write((const char*)idx.values.data(), sizeof(std::vector<T_val>::value_type)*count);
return stream;
}
///
/// Reads text index from a stream
///
/// \param[in] stream Input stream
/// \param[out] idx Text index
///
/// \returns The stream \p stream
///
template <class T_key, class T_val, class T_idx>
inline std::istream& operator >>(_In_ std::istream& stream, _Out_ ZRCola::textindex<T_key, T_val, T_idx> &idx)
{
unsigned __int32 count;
// Read text index count.
stream.read((char*)&count, sizeof(count));
if (!stream.good()) return stream;
if (count) {
// Read text index.
idx.resize(count);
stream.read((char*)idx.data(), sizeof(ZRCola::textindex<T_key, T_val, T_idx>::value_type)*count);
if (!stream.good()) return stream;
} else
idx.clear();
// Read keys count.
stream.read((char*)&count, sizeof(count));
if (!stream.good()) return stream;
if (count) {
// Read keys.
idx.keys.resize(count);
stream.read((char*)idx.keys.data(), sizeof(std::vector<T_key>::value_type)*count);
if (!stream.good()) return stream;
} else
idx.keys.clear();
// Read value count.
stream.read((char*)&count, sizeof(count));
if (!stream.good()) return stream;
if (count) {
// Read values.
idx.values.resize(count);
stream.read((char*)idx.values.data(), sizeof(std::vector<T_val>::value_type)*count);
} else
idx.values.clear();
return stream;
}
#pragma warning(pop)

122
lib/libZRCola/include/zrcola/language.h
View File

@@ -23,6 +23,7 @@
#include <stdex/idrec.h>
#include <istream>
#include <ostream>
#include <vector>
#include <string>
@@ -258,41 +259,113 @@ const ZRCola::recordid_t stdex::idrec::record<ZRCola::langchar_db, ZRCola::recor
const ZRCola::recordid_t stdex::idrec::record<ZRCola::language_db, ZRCola::recordid_t, ZRCola::recordsize_t, ZRCOLA_RECORD_ALIGN>::id = *(ZRCola::recordid_t*)"LNG";
///
/// Writes language character database to a stream
///
/// \param[in] stream Output stream
/// \param[in] db Language character database
///
/// \returns The stream \p stream
///
inline std::ostream& operator <<(_In_ std::ostream& stream, _In_ const ZRCola::langchar_db &db)
{
// Write character index.
if (stream.fail()) return stream;
stream << db.idxChr;
#ifdef ZRCOLA_LANGCHAR_LANG_IDX
// Write language index.
if (stream.fail()) return stream;
stream << db.idxLng;
#endif
// Write data count.
std::vector<unsigned __int16>::size_type data_count = db.data.size();
#if defined(_WIN64) || defined(__x86_64__) || defined(__ppc64__)
// 4G check
if (data_count > 0xffffffff) {
stream.setstate(std::ios_base::failbit);
return stream;
}
#endif
if (stream.fail()) return stream;
unsigned __int32 count = (unsigned __int32)data_count;
stream.write((const char*)&count, sizeof(count));
// Write data.
if (stream.fail()) return stream;
stream.write((const char*)db.data.data(), sizeof(unsigned __int16)*count);
return stream;
}
///
/// Reads language character database from a stream
///
/// \param[in] stream Input stream
/// \param[in ] stream Input stream
/// \param[out] db Language character database
///
/// \returns The stream \p stream
///
inline std::istream& operator >>(_In_ std::istream& stream, _Out_ ZRCola::langchar_db &db)
{
unsigned __int32 count;
// Read index count.
stream.read((char*)&count, sizeof(count));
if (!stream.good()) return stream;
// Read character index.
db.idxChr.resize(count);
stream.read((char*)db.idxChr.data(), sizeof(unsigned __int32)*count);
stream >> db.idxChr;
if (!stream.good()) return stream;
#ifdef ZRCOLA_LANGCHAR_LANG_IDX
// Read language index.
db.idxLng.resize(count);
stream.read((char*)db.idxLng.data(), sizeof(unsigned __int32)*count);
stream >> db.idxLng;
if (!stream.good()) return stream;
#endif
// Read data count.
unsigned __int32 count;
stream.read((char*)&count, sizeof(count));
if (!stream.good()) return stream;
// Read data.
db.data.resize(count);
stream.read((char*)db.data.data(), sizeof(unsigned __int16)*count);
if (count) {
// Read data.
db.data.resize(count);
stream.read((char*)db.data.data(), sizeof(unsigned __int16)*count);
} else
db.data.clear();
return stream;
}
///
/// Writes language database to a stream
///
/// \param[in] stream Output stream
/// \param[in] db Language database
///
/// \returns The stream \p stream
///
inline std::ostream& operator <<(_In_ std::ostream& stream, _In_ const ZRCola::language_db &db)
{
// Write language index.
if (stream.fail()) return stream;
stream << db.idxLng;
// Write data count.
std::vector<unsigned __int16>::size_type data_count = db.data.size();
#if defined(_WIN64) || defined(__x86_64__) || defined(__ppc64__)
// 4G check
if (data_count > 0xffffffff) {
stream.setstate(std::ios_base::failbit);
return stream;
}
#endif
if (stream.fail()) return stream;
unsigned __int32 count = (unsigned __int32)data_count;
stream.write((const char*)&count, sizeof(count));
// Write data.
if (stream.fail()) return stream;
stream.write((const char*)db.data.data(), sizeof(unsigned __int16)*count);
return stream;
}
@@ -301,31 +374,28 @@ inline std::istream& operator >>(_In_ std::istream& stream, _Out_ ZRCola::langch
///
/// Reads language database from a stream
///
/// \param[in] stream Input stream
/// \param[in ] stream Input stream
/// \param[out] db Language database
///
/// \returns The stream \p stream
///
inline std::istream& operator >>(_In_ std::istream& stream, _Out_ ZRCola::language_db &db)
{
unsigned __int32 count;
// Read index count.
stream.read((char*)&count, sizeof(count));
if (!stream.good()) return stream;
// Read language index.
db.idxLng.resize(count);
stream.read((char*)db.idxLng.data(), sizeof(unsigned __int32)*count);
stream >> db.idxLng;
if (!stream.good()) return stream;
// Read data count.
unsigned __int32 count;
stream.read((char*)&count, sizeof(count));
if (!stream.good()) return stream;
// Read data.
db.data.resize(count);
stream.read((char*)db.data.data(), sizeof(unsigned __int16)*count);
if (count) {
// Read data.
db.data.resize(count);
stream.read((char*)db.data.data(), sizeof(unsigned __int16)*count);
} else
db.data.clear();
return stream;
}

64
lib/libZRCola/include/zrcola/translate.h
View File

@@ -24,6 +24,7 @@
#include <stdex/idrec.h>
#include <istream>
#include <ostream>
#include <vector>
#include <string>
@@ -247,39 +248,74 @@ namespace ZRCola {
const ZRCola::recordid_t stdex::idrec::record<ZRCola::translation_db, ZRCola::recordid_t, ZRCola::recordsize_t, ZRCOLA_RECORD_ALIGN>::id = *(ZRCola::recordid_t*)"TRN";
///
/// Writes translation database to a stream
///
/// \param[in] stream Output stream
/// \param[in] db Translation database
///
/// \returns The stream \p stream
///
inline std::ostream& operator <<(_In_ std::ostream& stream, _In_ const ZRCola::translation_db &db)
{
// Write composition index.
if (stream.fail()) return stream;
stream << db.idxComp;
// Write decomposition index.
if (stream.fail()) return stream;
stream << db.idxDecomp;
// Write data count.
std::vector<unsigned __int16>::size_type data_count = db.data.size();
#if defined(_WIN64) || defined(__x86_64__) || defined(__ppc64__)
// 4G check
if (data_count > 0xffffffff) {
stream.setstate(std::ios_base::failbit);
return stream;
}
#endif
if (stream.fail()) return stream;
unsigned __int32 count = (unsigned __int32)data_count;
stream.write((const char*)&count, sizeof(count));
// Write data.
if (stream.fail()) return stream;
stream.write((const char*)db.data.data(), sizeof(unsigned __int16)*count);
return stream;
}
///
/// Reads translation database from a stream
///
/// \param[in] stream Input stream
/// \param[in ] stream Input stream
/// \param[out] db Translation database
///
/// \returns The stream \p stream
///
inline std::istream& operator >>(_In_ std::istream& stream, _Out_ ZRCola::translation_db &db)
{
unsigned __int32 count;
// Read index count.
stream.read((char*)&count, sizeof(count));
if (!stream.good()) return stream;
// Read composition index.
db.idxComp.resize(count);
stream.read((char*)db.idxComp.data(), sizeof(unsigned __int32)*count);
stream >> db.idxComp;
if (!stream.good()) return stream;
// Read decomposition index.
db.idxDecomp.resize(count);
stream.read((char*)db.idxDecomp.data(), sizeof(unsigned __int32)*count);
stream >> db.idxDecomp;
if (!stream.good()) return stream;
// Read data count.
unsigned __int32 count;
stream.read((char*)&count, sizeof(count));
if (!stream.good()) return stream;
// Read data.
db.data.resize(count);
stream.read((char*)db.data.data(), sizeof(unsigned __int16)*count);
if (count) {
// Read data.
db.data.resize(count);
stream.read((char*)db.data.data(), sizeof(unsigned __int16)*count);
} else
db.data.clear();
return stream;
}