Amebis/cld2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Dynamic mode cleanup: Use generated constants, cleanup TODOs. For more info:

Browse Source 
https://code.google.com/p/cld2/issues/detail?id=7


git-svn-id: https://cld2.googlecode.com/svn/trunk@154 b252ecd4-b096-bf77-eb8e-91563289f87e
This commit is contained in:
andrewhayden@google.com
2014-03-11 12:42:09 +00:00
parent a57e2d13c3
commit db75897057
18 changed files with 106 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
internal/cld2_dynamic_data.cc
View File

@@ -110,7 +110,9 @@ void dumpHeader(FileHeader* header) {
return false;\
}
bool verify(const CLD2::ScoringTables* realData, const CLD2::ScoringTables* loadedData) {
bool verify(const CLD2::ScoringTables* realData,
const Supplement* realSupplement,
const CLD2::ScoringTables* loadedData) {
const int NUM_TABLES = 7;
const CLD2::CLD2TableSummary* realTableSummaries[NUM_TABLES];
realTableSummaries[0] = realData->unigram_compat_obj;
@@ -159,7 +161,7 @@ bool verify(const CLD2::ScoringTables* realData, const CLD2::ScoringTables* load
if (DEBUG) std::cout << "verified." << std::endl;
if (DEBUG) std::cout << "Verifying kExpectedScore... ";
CHECK_MEM_EQUALS(kExpectedScore, 614*4); // TODO: Don't hardcode 614*4.
CHECK_MEM_EQUALS(kExpectedScore, realSupplement->lengthOf_kAvgDeltaOctaScore);
if (DEBUG) std::cout << "verified." << std::endl;
// 3. Each table
@@ -171,20 +173,7 @@ bool verify(const CLD2::ScoringTables* realData, const CLD2::ScoringTables* load
CLD2::uint32 bytesPerBucket = sizeof(CLD2::IndirectProbBucket4);
CLD2::uint32 numBuckets = realData->kCLDTableSize;
CLD2::uint32 tableSizeBytes = bytesPerBucket * numBuckets;
CLD2::uint32 indirectTableSizeBytes =
realData->kCLDTableSizeOne * sizeof(CLD2::uint32);
// XXX XXX XXX HACK HACK HACK FIXME FIXME FIXME
// XXX XXX XXX HACK HACK HACK FIXME FIXME FIXME
// XXX XXX XXX HACK HACK HACK FIXME FIXME FIXME
// cld2_generated_cjk_compatible.cc has a kCLDTableSizeOne of zero!
if (x == 0) { // cld2_generated_cjk_compatible.cc
indirectTableSizeBytes = 239*2*4;
}
// XXX XXX XXX HACK HACK HACK FIXME FIXME FIXME
// XXX XXX XXX HACK HACK HACK FIXME FIXME FIXME
// XXX XXX XXX HACK HACK HACK FIXME FIXME FIXME
CLD2::uint32 indirectTableSizeBytes = realSupplement->indirectTableSizes[x];
CLD2::uint32 recognizedScriptsSizeBytes =
strlen(realData->kRecognizedLangScripts) + 1; // null terminator included

23
internal/cld2_dynamic_data.h
View File

@@ -194,6 +194,19 @@ typedef struct {
TableHeader* tableHeaders;
} FileHeader;
// The CLD2::TableHeader structure doesn't contain everything that is needed
// to dump table data. Specifically, the size of the indirect table is not
// part of the data structure. Any such data are captured in this struct.
typedef struct {
const CLD2::uint32 lengthOf_kAvgDeltaOctaScore;
// An array of 32-bit unsigned integers representing the indirect table sizes
// for each of the table headers in the FileHeader structure. It is assumed
// that there is exactly one entry in the array for each table header present
// in the tableHeaders field of the FileHeader, and that they are in the
// same order as the entries in that structure.
const CLD2::uint32* indirectTableSizes;
} Supplement;
// Calculate the exact size of a header that encodes the specified number of
// tables. This can be used to reserve space within the data file,
// calculate offsets, and so on.
@@ -202,9 +215,13 @@ CLD2::uint32 calculateHeaderSize(CLD2::uint32 numTables);
// Dump a given header to stdout as a human-readable string.
void dumpHeader(FileHeader* header);
// Verify that a given pair of scoring tables match precisely
// If there is a problem, returns an error message; otherwise, the empty string.
bool verify(const CLD2::ScoringTables* realData, const CLD2::ScoringTables* loadedData);
// Verify that a given pair of scoring tables match precisely.
// Uses the provided supplement to verify information that cannot be otherwise
// checked from the CLD2::ScoringTables structure.
// If there is a problem, returns false.
bool verify(const CLD2::ScoringTables* realData,
const Supplement* realSupplement,
const CLD2::ScoringTables* loadedData);
// Return true iff the program is running in little-endian mode.
bool isLittleEndian();

37
internal/cld2_dynamic_data_extractor.cc
View File

@@ -45,7 +45,9 @@ void writeChunk(FILE *f, const void* data, CLD2::uint32 startAt, CLD2::uint32 le
fwrite(data, 1, length, f);
}
void writeDataFile(const CLD2::ScoringTables* data, const char* fileName) {
void writeDataFile(const CLD2::ScoringTables* data,
const CLD2DynamicData::Supplement* supplement,
const char* fileName) {
// The order here is hardcoded and MUST NOT BE CHANGED, else you will de-sync
// with the reading code.
const char ZERO = 0;
@@ -64,8 +66,8 @@ void writeDataFile(const CLD2::ScoringTables* data, const char* fileName) {
fileHeader.numTablesEncoded = NUM_TABLES;
fileHeader.tableHeaders = tableHeaders;
initUtf8Headers(&fileHeader, data->unigram_obj);
initDeltaHeaders(&fileHeader, data->kExpectedScore);
initTableHeaders(tableSummaries, NUM_TABLES, tableHeaders);
initDeltaHeaders(&fileHeader, supplement->lengthOf_kAvgDeltaOctaScore);
initTableHeaders(tableSummaries, NUM_TABLES, supplement, tableHeaders);
alignAll(&fileHeader, 16); // Align all sections to 128-bit boundaries
// We are ready to rock.
@@ -160,7 +162,12 @@ void writeDataFile(const CLD2::ScoringTables* data, const char* fileName) {
}
void initTableHeaders(const CLD2::CLD2TableSummary** summaries,
int numSummaries, CLD2DynamicData::TableHeader* tableHeaders) {
const int numSummaries,
const CLD2DynamicData::Supplement* supplement,
CLD2DynamicData::TableHeader* tableHeaders) {
// Important: As documented in the .h, we assume that the Supplement data
// structure contains exactly one entry in indirectTableSizes for each
// CLD2TableSummary, in the same order.
for (int x=0; x<numSummaries; x++) {
const CLD2::CLD2TableSummary* summary = summaries[x];
CLD2DynamicData::TableHeader& tableHeader = tableHeaders[x];
@@ -176,19 +183,7 @@ void initTableHeaders(const CLD2::CLD2TableSummary** summaries,
CLD2::uint32 numBuckets = summary->kCLDTableSize;
CLD2::uint32 tableSizeBytes = bytesPerBucket * numBuckets;
CLD2::uint32 indirectTableSizeBytes =
summary->kCLDTableSizeOne * sizeof(CLD2::uint32);
// XXX XXX XXX HACK HACK HACK FIXME FIXME FIXME
// XXX XXX XXX HACK HACK HACK FIXME FIXME FIXME
// XXX XXX XXX HACK HACK HACK FIXME FIXME FIXME
// cld2_generated_cjk_compatible.cc has a kCLDTableSizeOne of zero!
if (x == 0) { // cld2_generated_cjk_compatible.cc
indirectTableSizeBytes = 239*2*4;
}
// XXX XXX XXX HACK HACK HACK FIXME FIXME FIXME
// XXX XXX XXX HACK HACK HACK FIXME FIXME FIXME
// XXX XXX XXX HACK HACK HACK FIXME FIXME FIXME
supplement->indirectTableSizes[x] * sizeof(CLD2::uint32);
CLD2::uint32 recognizedScriptsSizeBytes =
strlen(summary->kRecognizedLangScripts) + 1; // note null terminator
@@ -203,7 +198,7 @@ void initTableHeaders(const CLD2::CLD2TableSummary** summaries,
// Assuming that all fields have been set in the specified header, re-align
// the starting positions of all data chunks to be aligned along 64-bit
// boundaries for maximum efficiency.
void alignAll(CLD2DynamicData::FileHeader* header, int alignment) {
void alignAll(CLD2DynamicData::FileHeader* header, const int alignment) {
CLD2::uint32 totalPadding = 0;
if (DEBUG) { std::cout << "Align for " << (alignment*8) << " bits." << std::endl; }
CLD2::uint32 headerSize = CLD2DynamicData::calculateHeaderSize(
@@ -264,7 +259,6 @@ void alignAll(CLD2DynamicData::FileHeader* header, int alignment) {
offset += header->lengthOf_kAvgDeltaOctaScore;
}
// TODO: The rest of the fields
for (int x=0; x<header->numTablesEncoded; x++) {
CLD2DynamicData::TableHeader& tableHeader = header->tableHeaders[x];
int tablePad = alignment - (offset % alignment);
@@ -347,10 +341,9 @@ void alignAll(CLD2DynamicData::FileHeader* header, int alignment) {
}
}
void initDeltaHeaders(CLD2DynamicData::FileHeader* header, const short* deltaArray) {
// TODO: Don't hardcode 614*4. Get constant from generated_language.cc?
void initDeltaHeaders(CLD2DynamicData::FileHeader* header, const CLD2::uint32 deltaLength) {
header->startOf_kAvgDeltaOctaScore = 0;
header->lengthOf_kAvgDeltaOctaScore = 614 * 4; // from cld_generated_score_quad_octa_1024_256.cc
header->lengthOf_kAvgDeltaOctaScore = deltaLength;
}
void initUtf8Headers(CLD2DynamicData::FileHeader* header, const CLD2::UTF8PropObj* utf8Object) {

16
internal/cld2_dynamic_data_extractor.h
View File

@@ -33,22 +33,28 @@ void initUtf8Headers(CLD2DynamicData::FileHeader* header,
// Populates all the AvgDeltaOctaScore-related fields of the header.
void initDeltaHeaders(CLD2DynamicData::FileHeader* header,
const short* deltaArray);
const CLD2::uint32 deltaLength);
// Populates all fields of all table headers for the specified table summaries.
// Tables are laid out back-to-back in the order that they are specified in the
// input array of summaries, and the headers are filled in in the same order.
// IMPORTANT: The Supplement data structure must contain exactly one entry in
// indirectTableSizes for each CLD2TableSummary in the summaries parameter,
// in the same order.
void initTableHeaders(const CLD2::CLD2TableSummary** summaries,
int numSummaries, CLD2DynamicData::TableHeader* tableSummaryHeaders);
const int numSummaries,
const CLD2DynamicData::Supplement* supplement,
CLD2DynamicData::TableHeader* tableSummaryHeaders);
// Align all entries in the data block along boundaries that are multiples of
// the specified number of bytes. For example, to align everything along 64-bit
// boundaries, pass an alignment of 8 (bytes).
void alignAll(CLD2DynamicData::FileHeader* header, int alignment);
void alignAll(CLD2DynamicData::FileHeader* header, const int alignment);
// Write the dynamic data file to disk.
void writeDataFile(const CLD2::ScoringTables* data, const char* fileName);
void writeDataFile(const CLD2::ScoringTables* data,
const CLD2DynamicData::Supplement* supplement,
const char* fileName);
} // End namespace CLD2DynamicDataExtractor
#endif // CLD2_INTERNAL_CLD2_DYNAMIC_DATA_EXTRACTOR_H_

23
internal/cld2_dynamic_data_tool.cc
View File

@@ -40,6 +40,14 @@ namespace CLD2 {
extern const CLD2TableSummary kDeltaOcta_obj;
extern const CLD2TableSummary kDistinctOcta_obj;
extern const short kAvgDeltaOctaScore[];
extern const uint32 kAvgDeltaOctaScoreSize;
extern const uint32 kCompatTableIndSize;
extern const uint32 kCjkDeltaBiIndSize;
extern const uint32 kDistinctBiTableIndSize;
extern const uint32 kQuadChromeIndSize;
extern const uint32 kQuadChrome2IndSize;
extern const uint32 kDeltaOctaIndSize;
extern const uint32 kDistinctOctaIndSize;
}
int main(int argc, char** argv) {
@@ -123,9 +131,23 @@ Usage:\n\
&CLD2::kDistinctOcta_obj,
CLD2::kAvgDeltaOctaScore,
};
const CLD2::uint32 indirectTableSizes[7] = {
CLD2::kCompatTableIndSize,
CLD2::kCjkDeltaBiIndSize,
CLD2::kDistinctBiTableIndSize,
CLD2::kQuadChromeIndSize,
CLD2::kQuadChrome2IndSize,
CLD2::kDeltaOctaIndSize,
CLD2::kDistinctOctaIndSize
};
const CLD2DynamicData::Supplement supplement = {
CLD2::kAvgDeltaOctaScoreSize,
indirectTableSizes
};
if (mode == 1) { // dump
CLD2DynamicDataExtractor::writeDataFile(
static_cast<const CLD2::ScoringTables*>(&realData),
&supplement,
fileName);
} else if (mode == 3) { // head
CLD2DynamicData::FileHeader* header = CLD2DynamicDataLoader::loadHeaderFromFile(fileName);
@@ -149,6 +171,7 @@ Usage:\n\
}
bool result = CLD2DynamicData::verify(
static_cast<const CLD2::ScoringTables*>(&realData),
&supplement,
static_cast<const CLD2::ScoringTables*>(loadedData));
CLD2DynamicDataLoader::unloadDataFile(&loadedData, &mmapAddress, &mmapLength);
if (loadedData != NULL || mmapAddress != NULL || mmapLength != 0) {

2
internal/cld2_generated_cjk_compatible.cc
View File

@@ -37,7 +37,7 @@ static const IndirectProbBucket4 kCompatTable[kCompatTableSize] = {
// This is all part of using one-byte mappings for CJK but wanting to
// convert them to normal langprob values to share the scoring code.
static const uint32 kCompatTableSizeOne = 0; // One-langprob count
static const uint32 kCompatTableIndSize = 239 * 2; // Largest subscript
extern const uint32 kCompatTableIndSize = 239 * 2; // Largest subscript
static const uint32 kCompatTableInd[kCompatTableIndSize] = {
// [0000]
0x00000000, 0x00000000, // [0] zh.0 zhT.0 ja.0 ko.0 vi.0 za.0

10
internal/cld2_generated_deltaocta0122.cc
View File

@@ -65841,8 +65841,10 @@ static const IndirectProbBucket4 kDeltaOcta0122[kDeltaOcta0122Size] = {
};
// table_hash = 3043-1366, unused_entries = 13446 (5.13%)
static const uint32 kDeltaOcta0122SizeOne = 2143; // Bucket count one-lang
static const uint32 kDeltaOcta0122Ind[2143] = {
static const uint32 kDeltaOcta0122SizeOne = 2143; // Bucket count one-lang
static const uint32 kDeltaOcta0122IndSize = 2143; // Largest subscript
extern const uint32 kDeltaOctaIndSize = kDeltaOcta0122IndSize; // Largest subscript
static const uint32 kDeltaOcta0122Ind[kDeltaOcta0122IndSize] = {
// [0000]
0x00000000, 0x00000000, 0x1600171b, 0x0000031c, // -- -- sr.hr.un_770 nl.un.un_800
0x00005337, 0x00003e42, 0x1e001c1b, 0x00004537, // ht.un.un_B00 rm.un.un_C00 id.ms.un_770 ln.un.un_B00
@@ -66540,7 +66542,9 @@ static const IndirectProbBucket4 kDeltaOcta0122_2[kDeltaOcta0122_2Size] = {
// table_hash = ffff-ffff, unused_entries = 0 (0.00%)
static const uint32 kDeltaOcta0122_2SizeOne = 2; // Bucket count one-lang
static const uint32 kDeltaOcta0122_2Ind[2] = {
static const uint32 kDeltaOcta0122_2IndSize = 2; // Largest subscript
extern const uint32 kDeltaOcta2IndSize = kDeltaOcta0122_2IndSize; // Source-agnostic named constant
static const uint32 kDeltaOcta0122_2Ind[kDeltaOcta0122_2IndSize] = {
// [0000] --- double_langprob_start=0002 ---
0x00000000, 0x00000000, // -- --
//

2
internal/cld2_generated_deltaoctachrome0122.cc
View File

@@ -4255,6 +4255,7 @@ static const IndirectProbBucket4 kDeltaOctaChrome0122[kDeltaOctaChrome0122Size]
static const uint32 kDeltaOctaChrome0122SizeOne = 955; // One-langprob count
static const uint32 kDeltaOctaChrome0122IndSize = 955; // Largest subscript
extern const uint32 kDeltaOctaIndSize = kDeltaOctaChrome0122IndSize; // Source-agnostic named constant
static const uint32 kDeltaOctaChrome0122Ind[kDeltaOctaChrome0122IndSize] = {
// [0000]
0x00000000, 0x00000000, 0x00001324, 0x00000e1c, // -- -- bh.un.un_900 is.un.un_800
@@ -4580,6 +4581,7 @@ static const IndirectProbBucket4 kDeltaOctaChrome0122_2[kDeltaOctaChrome0122_2Si
static const uint32 kDeltaOctaChrome0122_2SizeOne = 2; // One-langprob count
static const uint32 kDeltaOctaChrome0122_2IndSize = 2; // Largest subscript
extern const uint32 kDeltaOcta2IndSize = kDeltaOctaChrome0122_2IndSize; // Source-agnostic named constant
static const uint32 kDeltaOctaChrome0122_2Ind[kDeltaOctaChrome0122_2IndSize] = {
// [0000] --- double_langprob_start=0002 ---
0x00000000, 0x00000000, // -- --

12
internal/cld2_generated_distinctocta0122.cc
View File

@@ -8284,8 +8284,10 @@ static const IndirectProbBucket4 kDistinctOcta0122[kDistinctOcta0122Size] = {
};
// table_hash = 981b-55ce, unused_entries = 14207 (43.36%)
static const uint32 kDistinctOcta0122SizeOne = 98; // Bucket count one-lang
static const uint32 kDistinctOcta0122Ind[98] = {
static const uint32 kDistinctOcta0122SizeOne = 98; // Bucket count one-lang
static const uint32 kDistinctOcta0122IndSize = 98; // Largest subscript
extern const uint32 kDistinctOctaIndSize = kDistinctOcta0122IndSize; // Source-agnostic named constant
static const uint32 kDistinctOcta0122Ind[kDistinctOcta0122IndSize] = {
// [0000]
0x00000000, 0x00000000, 0x00001706, 0x00002a06, // -- -- sr.un.un_400 bo.un.un_400
0x00003406, 0x00003303, 0x00003906, 0x00001e01, // xh.un.un_400 nn.un.un_300 dz.un.un_400 ms.un.un_200
@@ -8341,8 +8343,10 @@ static const IndirectProbBucket4 kDistinctOcta0122_2[kDistinctOcta0122_2Size] =
};
// table_hash = ffff-ffff, unused_entries = 0 (0.00%)
static const uint32 kDistinctOcta0122_2SizeOne = 2; // Bucket count one-lang
static const uint32 kDistinctOcta0122_2Ind[2] = {
static const uint32 kDistinctOcta0122_2SizeOne = 2; // Bucket count one-lang
static const uint32 kDistinctOcta0122_2IndSize = 2; // Largest subscript
extern const uint32 kDistinctOcta2IndSize = kDistinctOcta0122_2IndSize; // Source-agnostic named constant
static const uint32 kDistinctOcta0122_2Ind[kDistinctOcta0122_2IndSize] = {
// [0000] --- double_langprob_start=0002 ---
0x00000000, 0x00000000, // -- --
//

2
internal/cld2_generated_distinctoctachrome0122.cc
View File

@@ -2137,6 +2137,7 @@ static const IndirectProbBucket4 kDistinctOctaChrome0122[kDistinctOctaChrome0122
static const uint32 kDistinctOctaChrome0122SizeOne = 76; // One-langprob count
static const uint32 kDistinctOctaChrome0122IndSize = 76; // Largest subscript
extern const uint32 kDistinctOctaIndSize = kDistinctOctaChrome0122IndSize; // Source-agnostic named constant
static const uint32 kDistinctOctaChrome0122Ind[kDistinctOctaChrome0122IndSize] = {
// [0000]
0x00000000, 0x00000000, 0x00000b03, 0x00001c03, // -- -- es.un.un_300 id.un.un_300
@@ -2187,6 +2188,7 @@ static const IndirectProbBucket4 kDistinctOctaChrome0122_2[kDistinctOctaChrome01
static const uint32 kDistinctOctaChrome0122_2SizeOne = 2; // One-langprob count
static const uint32 kDistinctOctaChrome0122_2IndSize = 2; // Largest subscript
extern const uint32 kDistinctOcta2IndSize = kDistinctOctaChrome0122_2IndSize; // Source-agnostic named constant
static const uint32 kDistinctOctaChrome0122_2Ind[kDistinctOctaChrome0122_2IndSize] = {
// [0000] --- double_langprob_start=0002 ---
0x00000000, 0x00000000, // -- --

2
internal/cld2_generated_quadchrome0122_16.cc
View File

@@ -32952,6 +32952,7 @@ static const IndirectProbBucket4 kQuadChrome0122_16[kQuadChrome0122_16Size] = {
static const uint32 kQuadChrome0122_16SizeOne = 28727; // One-langprob count
static const uint32 kQuadChrome0122_16IndSize = 28727; // Largest subscript
extern const uint32 kQuadChromeIndSize = kQuadChrome0122_16IndSize; // Source-agnostic named constant
static const uint32 kQuadChrome0122_16Ind[kQuadChrome0122_16IndSize] = {
// [0000]
0x00000000, 0x00000000, 0x08040709, 0x00000a24, // -- -- bg.ru.uk_444 mk.un.un_900
@@ -50184,6 +50185,7 @@ static const IndirectProbBucket4 kQuadChrome0122_16_2[kQuadChrome0122_16_2Size]
static const uint32 kQuadChrome0122_16_2SizeOne = 8108; // One-langprob count
static const uint32 kQuadChrome0122_16_2IndSize = 8108; // Largest subscript
extern const uint32 kQuadChrome2IndSize = kQuadChrome0122_16_2IndSize; // Source-agnostic named constant
static const uint32 kQuadChrome0122_16_2Ind[kQuadChrome0122_16_2IndSize] = {
// [0000]
0x00000000, 0x00000000, 0x080c03a7, 0x20005504, // -- -- nl.sv.no_532 rw.sq.un_320

4
internal/cld2_generated_quadchrome0122_19.cc
View File

@@ -32952,6 +32952,7 @@ static const IndirectProbBucket4 kQuadChrome0122_19[kQuadChrome0122_19Size] = {
static const uint32 kQuadChrome0122_19SizeOne = 31722; // One-langprob count
static const uint32 kQuadChrome0122_19IndSize = 31722; // Largest subscript
extern const uint32 kQuadChromeIndSize = kQuadChrome0122_19IndSize; // Source-agnostic named constant
static const uint32 kQuadChrome0122_19Ind[kQuadChrome0122_19IndSize] = {
// [0000]
0x00000000, 0x00000000, 0x08040709, 0x00000a24, // -- -- bg.ru.uk_444 mk.un.un_900
@@ -59320,8 +59321,9 @@ static const IndirectProbBucket4 kQuadChrome0122_19_2[kQuadChrome0122_19_2Size]
};
// table_hash = 8afb-a95b, unused_entries = 75 (0.04%)
static const uint32 kQuadChrome0122_19_2SizeOne = 13863; // One-langprob count
static const uint32 kQuadChrome0122_19_2SizeOne = 13863; // One-langprob count
static const uint32 kQuadChrome0122_19_2IndSize = 13863; // Largest subscript
extern const uint32 kQuadChrome2IndSize = kQuadChrome0122_19_2IndSize; // Source-agnostic named constant
static const uint32 kQuadChrome0122_19_2Ind[kQuadChrome0122_19_2IndSize] = {
// [0000]
0x00000000, 0x00000000, 0x080c03a7, 0x20005504, // -- -- nl.sv.no_532 rw.sq.un_320

2
internal/cld2_generated_quadchrome0122_2.cc
View File

@@ -65750,6 +65750,7 @@ static const IndirectProbBucket4 kQuadChrome0122_2[kQuadChrome0122_2Size] = {
static const uint32 kQuadChrome0122_2SizeOne = 50708; // One-langprob count
static const uint32 kQuadChrome0122_2IndSize = 50708; // Largest subscript
extern const uint32 kQuadChromeIndSize = kQuadChrome0122_2IndSize; // Source-agnostic named constant
static const uint32 kQuadChrome0122_2Ind[kQuadChrome0122_2IndSize] = {
// [0000]
0x00000000, 0x00000000, 0x08040709, 0x06000c13, // -- -- bg.ru.uk_444 sv.de.un_650
@@ -81672,6 +81673,7 @@ static const IndirectProbBucket4 kQuadChrome0122_2_2[kQuadChrome0122_2_2Size] =
static const uint32 kQuadChrome0122_2_2SizeOne = 2; // One-langprob count
static const uint32 kQuadChrome0122_2_2IndSize = 2; // Largest subscript
extern const uint32 kQuadChrome2IndSize = kQuadChrome0122_2_2IndSize; // Source-agnostic named constant
static const uint32 kQuadChrome0122_2_2Ind[kQuadChrome0122_2_2IndSize] = {
// [0000] --- double_langprob_start=0002 ---
0x00000000, 0x00000000, // -- --

2
internal/cld_generated_cjk_delta_bi_32.cc
View File

@@ -8252,7 +8252,7 @@ static const IndirectProbBucket4 kCjkDeltaBi[kCjkDeltaBiSize] = {
};
static const uint32 kCjkDeltaBiSizeOne = 33; // One-langprob count
static const uint32 kCjkDeltaBiIndSize = 33; // Largest subscript
extern const uint32 kCjkDeltaBiIndSize = 33; // Largest subscript
static const uint32 kCjkDeltaBiInd[kCjkDeltaBiIndSize] = {
// [0000]
0x00000000, 0x00001d2d, 0x00001d1c, 0x00000000,

2
internal/cld_generated_cjk_delta_bi_4.cc
View File

@@ -1102,7 +1102,7 @@ static const IndirectProbBucket4 kCjkDeltaBi[kCjkDeltaBiSize] = {
// table_hash = 860b-1885, unused_entries = 1630 (39.79%)
static const uint32 kCjkDeltaBiSizeOne = 34; // One-langprob count
static const uint32 kCjkDeltaBiIndSize = 34; // Largest subscript
extern const uint32 kCjkDeltaBiIndSize = 34; // Largest subscript
static const uint32 kCjkDeltaBiInd[kCjkDeltaBiIndSize] = {
// [0000]
0x00000000, 0x00000000, 0x00001d1c, 0x00000242, // -- -- zh-Hant.un.un_800 ja.un.un_C00

2
internal/cld_generated_score_quad_octa_0122.cc
View File

@@ -25,7 +25,7 @@
namespace CLD2 {
// Average score per 1024 bytes
static const int kAvgDeltaOctaScoreSize = 614 * 4;
extern const int kAvgDeltaOctaScoreSize = 614 * 4;
extern const short kAvgDeltaOctaScore[kAvgDeltaOctaScoreSize] = {
// Latn Cyrl Arab Other script
// Updated 20140204 for CLD2 full

2
internal/cld_generated_score_quad_octa_0122_2.cc
View File

@@ -16,7 +16,7 @@
namespace CLD2 {
// Average score per 1024 bytes
static const int kAvgDeltaOctaScoreSize = 614 * 4;
extern const int kAvgDeltaOctaScoreSize = 614 * 4;
extern const short kAvgDeltaOctaScore[kAvgDeltaOctaScoreSize] = {
// Latn Cyrl Arab Other script
// Updated 20140202 for CLD2 Chrome 256K entries

2
internal/generated_distinct_bi_0.cc
View File

@@ -32,7 +32,7 @@ static const IndirectProbBucket4 kDistinctBiTable[kDistinctBiTableSize] = {
};
static const uint32 kDistinctBiTableSizeOne = 1; // One-langprob count
static const uint32 kDistinctBiTableIndSize = 1; // Largest subscript
extern const uint32 kDistinctBiTableIndSize = 1; // Largest subscript
static const uint32 kDistinctBiTableInd[kDistinctBiTableIndSize] = {
// [0000]
0x00000000, };