diff --git a/internal/cld2_dynamic_data.cc b/internal/cld2_dynamic_data.cc index b867344..1655170 100644 --- a/internal/cld2_dynamic_data.cc +++ b/internal/cld2_dynamic_data.cc @@ -110,7 +110,9 @@ void dumpHeader(FileHeader* header) { return false;\ } -bool verify(const CLD2::ScoringTables* realData, const CLD2::ScoringTables* loadedData) { +bool verify(const CLD2::ScoringTables* realData, + const Supplement* realSupplement, + const CLD2::ScoringTables* loadedData) { const int NUM_TABLES = 7; const CLD2::CLD2TableSummary* realTableSummaries[NUM_TABLES]; realTableSummaries[0] = realData->unigram_compat_obj; @@ -159,7 +161,7 @@ bool verify(const CLD2::ScoringTables* realData, const CLD2::ScoringTables* load if (DEBUG) std::cout << "verified." << std::endl; if (DEBUG) std::cout << "Verifying kExpectedScore... "; - CHECK_MEM_EQUALS(kExpectedScore, 614*4); // TODO: Don't hardcode 614*4. + CHECK_MEM_EQUALS(kExpectedScore, realSupplement->lengthOf_kAvgDeltaOctaScore); if (DEBUG) std::cout << "verified." << std::endl; // 3. Each table @@ -171,20 +173,7 @@ bool verify(const CLD2::ScoringTables* realData, const CLD2::ScoringTables* load CLD2::uint32 bytesPerBucket = sizeof(CLD2::IndirectProbBucket4); CLD2::uint32 numBuckets = realData->kCLDTableSize; CLD2::uint32 tableSizeBytes = bytesPerBucket * numBuckets; - CLD2::uint32 indirectTableSizeBytes = - realData->kCLDTableSizeOne * sizeof(CLD2::uint32); - - // XXX XXX XXX HACK HACK HACK FIXME FIXME FIXME - // XXX XXX XXX HACK HACK HACK FIXME FIXME FIXME - // XXX XXX XXX HACK HACK HACK FIXME FIXME FIXME - // cld2_generated_cjk_compatible.cc has a kCLDTableSizeOne of zero! - if (x == 0) { // cld2_generated_cjk_compatible.cc - indirectTableSizeBytes = 239*2*4; - } - // XXX XXX XXX HACK HACK HACK FIXME FIXME FIXME - // XXX XXX XXX HACK HACK HACK FIXME FIXME FIXME - // XXX XXX XXX HACK HACK HACK FIXME FIXME FIXME - + CLD2::uint32 indirectTableSizeBytes = realSupplement->indirectTableSizes[x]; CLD2::uint32 recognizedScriptsSizeBytes = strlen(realData->kRecognizedLangScripts) + 1; // null terminator included diff --git a/internal/cld2_dynamic_data.h b/internal/cld2_dynamic_data.h index 693d35b..ea929d7 100644 --- a/internal/cld2_dynamic_data.h +++ b/internal/cld2_dynamic_data.h @@ -194,6 +194,19 @@ typedef struct { TableHeader* tableHeaders; } FileHeader; +// The CLD2::TableHeader structure doesn't contain everything that is needed +// to dump table data. Specifically, the size of the indirect table is not +// part of the data structure. Any such data are captured in this struct. +typedef struct { + const CLD2::uint32 lengthOf_kAvgDeltaOctaScore; + // An array of 32-bit unsigned integers representing the indirect table sizes + // for each of the table headers in the FileHeader structure. It is assumed + // that there is exactly one entry in the array for each table header present + // in the tableHeaders field of the FileHeader, and that they are in the + // same order as the entries in that structure. + const CLD2::uint32* indirectTableSizes; +} Supplement; + // Calculate the exact size of a header that encodes the specified number of // tables. This can be used to reserve space within the data file, // calculate offsets, and so on. @@ -202,9 +215,13 @@ CLD2::uint32 calculateHeaderSize(CLD2::uint32 numTables); // Dump a given header to stdout as a human-readable string. void dumpHeader(FileHeader* header); -// Verify that a given pair of scoring tables match precisely -// If there is a problem, returns an error message; otherwise, the empty string. -bool verify(const CLD2::ScoringTables* realData, const CLD2::ScoringTables* loadedData); +// Verify that a given pair of scoring tables match precisely. +// Uses the provided supplement to verify information that cannot be otherwise +// checked from the CLD2::ScoringTables structure. +// If there is a problem, returns false. +bool verify(const CLD2::ScoringTables* realData, + const Supplement* realSupplement, + const CLD2::ScoringTables* loadedData); // Return true iff the program is running in little-endian mode. bool isLittleEndian(); diff --git a/internal/cld2_dynamic_data_extractor.cc b/internal/cld2_dynamic_data_extractor.cc index 36ce41e..39c8340 100644 --- a/internal/cld2_dynamic_data_extractor.cc +++ b/internal/cld2_dynamic_data_extractor.cc @@ -45,7 +45,9 @@ void writeChunk(FILE *f, const void* data, CLD2::uint32 startAt, CLD2::uint32 le fwrite(data, 1, length, f); } -void writeDataFile(const CLD2::ScoringTables* data, const char* fileName) { +void writeDataFile(const CLD2::ScoringTables* data, + const CLD2DynamicData::Supplement* supplement, + const char* fileName) { // The order here is hardcoded and MUST NOT BE CHANGED, else you will de-sync // with the reading code. const char ZERO = 0; @@ -64,8 +66,8 @@ void writeDataFile(const CLD2::ScoringTables* data, const char* fileName) { fileHeader.numTablesEncoded = NUM_TABLES; fileHeader.tableHeaders = tableHeaders; initUtf8Headers(&fileHeader, data->unigram_obj); - initDeltaHeaders(&fileHeader, data->kExpectedScore); - initTableHeaders(tableSummaries, NUM_TABLES, tableHeaders); + initDeltaHeaders(&fileHeader, supplement->lengthOf_kAvgDeltaOctaScore); + initTableHeaders(tableSummaries, NUM_TABLES, supplement, tableHeaders); alignAll(&fileHeader, 16); // Align all sections to 128-bit boundaries // We are ready to rock. @@ -160,7 +162,12 @@ void writeDataFile(const CLD2::ScoringTables* data, const char* fileName) { } void initTableHeaders(const CLD2::CLD2TableSummary** summaries, - int numSummaries, CLD2DynamicData::TableHeader* tableHeaders) { + const int numSummaries, + const CLD2DynamicData::Supplement* supplement, + CLD2DynamicData::TableHeader* tableHeaders) { + // Important: As documented in the .h, we assume that the Supplement data + // structure contains exactly one entry in indirectTableSizes for each + // CLD2TableSummary, in the same order. for (int x=0; xkCLDTableSize; CLD2::uint32 tableSizeBytes = bytesPerBucket * numBuckets; CLD2::uint32 indirectTableSizeBytes = - summary->kCLDTableSizeOne * sizeof(CLD2::uint32); - - // XXX XXX XXX HACK HACK HACK FIXME FIXME FIXME - // XXX XXX XXX HACK HACK HACK FIXME FIXME FIXME - // XXX XXX XXX HACK HACK HACK FIXME FIXME FIXME - // cld2_generated_cjk_compatible.cc has a kCLDTableSizeOne of zero! - if (x == 0) { // cld2_generated_cjk_compatible.cc - indirectTableSizeBytes = 239*2*4; - } - // XXX XXX XXX HACK HACK HACK FIXME FIXME FIXME - // XXX XXX XXX HACK HACK HACK FIXME FIXME FIXME - // XXX XXX XXX HACK HACK HACK FIXME FIXME FIXME - + supplement->indirectTableSizes[x] * sizeof(CLD2::uint32); CLD2::uint32 recognizedScriptsSizeBytes = strlen(summary->kRecognizedLangScripts) + 1; // note null terminator @@ -203,7 +198,7 @@ void initTableHeaders(const CLD2::CLD2TableSummary** summaries, // Assuming that all fields have been set in the specified header, re-align // the starting positions of all data chunks to be aligned along 64-bit // boundaries for maximum efficiency. -void alignAll(CLD2DynamicData::FileHeader* header, int alignment) { +void alignAll(CLD2DynamicData::FileHeader* header, const int alignment) { CLD2::uint32 totalPadding = 0; if (DEBUG) { std::cout << "Align for " << (alignment*8) << " bits." << std::endl; } CLD2::uint32 headerSize = CLD2DynamicData::calculateHeaderSize( @@ -264,7 +259,6 @@ void alignAll(CLD2DynamicData::FileHeader* header, int alignment) { offset += header->lengthOf_kAvgDeltaOctaScore; } - // TODO: The rest of the fields for (int x=0; xnumTablesEncoded; x++) { CLD2DynamicData::TableHeader& tableHeader = header->tableHeaders[x]; int tablePad = alignment - (offset % alignment); @@ -347,10 +341,9 @@ void alignAll(CLD2DynamicData::FileHeader* header, int alignment) { } } -void initDeltaHeaders(CLD2DynamicData::FileHeader* header, const short* deltaArray) { - // TODO: Don't hardcode 614*4. Get constant from generated_language.cc? +void initDeltaHeaders(CLD2DynamicData::FileHeader* header, const CLD2::uint32 deltaLength) { header->startOf_kAvgDeltaOctaScore = 0; - header->lengthOf_kAvgDeltaOctaScore = 614 * 4; // from cld_generated_score_quad_octa_1024_256.cc + header->lengthOf_kAvgDeltaOctaScore = deltaLength; } void initUtf8Headers(CLD2DynamicData::FileHeader* header, const CLD2::UTF8PropObj* utf8Object) { diff --git a/internal/cld2_dynamic_data_extractor.h b/internal/cld2_dynamic_data_extractor.h index 8850dd7..614d023 100644 --- a/internal/cld2_dynamic_data_extractor.h +++ b/internal/cld2_dynamic_data_extractor.h @@ -33,22 +33,28 @@ void initUtf8Headers(CLD2DynamicData::FileHeader* header, // Populates all the AvgDeltaOctaScore-related fields of the header. void initDeltaHeaders(CLD2DynamicData::FileHeader* header, - const short* deltaArray); + const CLD2::uint32 deltaLength); // Populates all fields of all table headers for the specified table summaries. // Tables are laid out back-to-back in the order that they are specified in the // input array of summaries, and the headers are filled in in the same order. +// IMPORTANT: The Supplement data structure must contain exactly one entry in +// indirectTableSizes for each CLD2TableSummary in the summaries parameter, +// in the same order. void initTableHeaders(const CLD2::CLD2TableSummary** summaries, - int numSummaries, CLD2DynamicData::TableHeader* tableSummaryHeaders); + const int numSummaries, + const CLD2DynamicData::Supplement* supplement, + CLD2DynamicData::TableHeader* tableSummaryHeaders); // Align all entries in the data block along boundaries that are multiples of // the specified number of bytes. For example, to align everything along 64-bit // boundaries, pass an alignment of 8 (bytes). -void alignAll(CLD2DynamicData::FileHeader* header, int alignment); +void alignAll(CLD2DynamicData::FileHeader* header, const int alignment); // Write the dynamic data file to disk. -void writeDataFile(const CLD2::ScoringTables* data, const char* fileName); - +void writeDataFile(const CLD2::ScoringTables* data, + const CLD2DynamicData::Supplement* supplement, + const char* fileName); } // End namespace CLD2DynamicDataExtractor #endif // CLD2_INTERNAL_CLD2_DYNAMIC_DATA_EXTRACTOR_H_ diff --git a/internal/cld2_dynamic_data_tool.cc b/internal/cld2_dynamic_data_tool.cc index de97f88..ebb8474 100644 --- a/internal/cld2_dynamic_data_tool.cc +++ b/internal/cld2_dynamic_data_tool.cc @@ -40,6 +40,14 @@ namespace CLD2 { extern const CLD2TableSummary kDeltaOcta_obj; extern const CLD2TableSummary kDistinctOcta_obj; extern const short kAvgDeltaOctaScore[]; + extern const uint32 kAvgDeltaOctaScoreSize; + extern const uint32 kCompatTableIndSize; + extern const uint32 kCjkDeltaBiIndSize; + extern const uint32 kDistinctBiTableIndSize; + extern const uint32 kQuadChromeIndSize; + extern const uint32 kQuadChrome2IndSize; + extern const uint32 kDeltaOctaIndSize; + extern const uint32 kDistinctOctaIndSize; } int main(int argc, char** argv) { @@ -123,9 +131,23 @@ Usage:\n\ &CLD2::kDistinctOcta_obj, CLD2::kAvgDeltaOctaScore, }; + const CLD2::uint32 indirectTableSizes[7] = { + CLD2::kCompatTableIndSize, + CLD2::kCjkDeltaBiIndSize, + CLD2::kDistinctBiTableIndSize, + CLD2::kQuadChromeIndSize, + CLD2::kQuadChrome2IndSize, + CLD2::kDeltaOctaIndSize, + CLD2::kDistinctOctaIndSize + }; + const CLD2DynamicData::Supplement supplement = { + CLD2::kAvgDeltaOctaScoreSize, + indirectTableSizes + }; if (mode == 1) { // dump CLD2DynamicDataExtractor::writeDataFile( static_cast(&realData), + &supplement, fileName); } else if (mode == 3) { // head CLD2DynamicData::FileHeader* header = CLD2DynamicDataLoader::loadHeaderFromFile(fileName); @@ -149,6 +171,7 @@ Usage:\n\ } bool result = CLD2DynamicData::verify( static_cast(&realData), + &supplement, static_cast(loadedData)); CLD2DynamicDataLoader::unloadDataFile(&loadedData, &mmapAddress, &mmapLength); if (loadedData != NULL || mmapAddress != NULL || mmapLength != 0) { diff --git a/internal/cld2_generated_cjk_compatible.cc b/internal/cld2_generated_cjk_compatible.cc index d08b66c..1ee5d88 100644 --- a/internal/cld2_generated_cjk_compatible.cc +++ b/internal/cld2_generated_cjk_compatible.cc @@ -37,7 +37,7 @@ static const IndirectProbBucket4 kCompatTable[kCompatTableSize] = { // This is all part of using one-byte mappings for CJK but wanting to // convert them to normal langprob values to share the scoring code. static const uint32 kCompatTableSizeOne = 0; // One-langprob count -static const uint32 kCompatTableIndSize = 239 * 2; // Largest subscript +extern const uint32 kCompatTableIndSize = 239 * 2; // Largest subscript static const uint32 kCompatTableInd[kCompatTableIndSize] = { // [0000] 0x00000000, 0x00000000, // [0] zh.0 zhT.0 ja.0 ko.0 vi.0 za.0 diff --git a/internal/cld2_generated_deltaocta0122.cc b/internal/cld2_generated_deltaocta0122.cc index 0e79690..f9f137b 100644 --- a/internal/cld2_generated_deltaocta0122.cc +++ b/internal/cld2_generated_deltaocta0122.cc @@ -65841,8 +65841,10 @@ static const IndirectProbBucket4 kDeltaOcta0122[kDeltaOcta0122Size] = { }; // table_hash = 3043-1366, unused_entries = 13446 (5.13%) -static const uint32 kDeltaOcta0122SizeOne = 2143; // Bucket count one-lang -static const uint32 kDeltaOcta0122Ind[2143] = { +static const uint32 kDeltaOcta0122SizeOne = 2143; // Bucket count one-lang +static const uint32 kDeltaOcta0122IndSize = 2143; // Largest subscript +extern const uint32 kDeltaOctaIndSize = kDeltaOcta0122IndSize; // Largest subscript +static const uint32 kDeltaOcta0122Ind[kDeltaOcta0122IndSize] = { // [0000] 0x00000000, 0x00000000, 0x1600171b, 0x0000031c, // -- -- sr.hr.un_770 nl.un.un_800 0x00005337, 0x00003e42, 0x1e001c1b, 0x00004537, // ht.un.un_B00 rm.un.un_C00 id.ms.un_770 ln.un.un_B00 @@ -66540,7 +66542,9 @@ static const IndirectProbBucket4 kDeltaOcta0122_2[kDeltaOcta0122_2Size] = { // table_hash = ffff-ffff, unused_entries = 0 (0.00%) static const uint32 kDeltaOcta0122_2SizeOne = 2; // Bucket count one-lang -static const uint32 kDeltaOcta0122_2Ind[2] = { +static const uint32 kDeltaOcta0122_2IndSize = 2; // Largest subscript +extern const uint32 kDeltaOcta2IndSize = kDeltaOcta0122_2IndSize; // Source-agnostic named constant +static const uint32 kDeltaOcta0122_2Ind[kDeltaOcta0122_2IndSize] = { // [0000] --- double_langprob_start=0002 --- 0x00000000, 0x00000000, // -- -- // diff --git a/internal/cld2_generated_deltaoctachrome0122.cc b/internal/cld2_generated_deltaoctachrome0122.cc index 035e8fb..c1c19cd 100644 --- a/internal/cld2_generated_deltaoctachrome0122.cc +++ b/internal/cld2_generated_deltaoctachrome0122.cc @@ -4255,6 +4255,7 @@ static const IndirectProbBucket4 kDeltaOctaChrome0122[kDeltaOctaChrome0122Size] static const uint32 kDeltaOctaChrome0122SizeOne = 955; // One-langprob count static const uint32 kDeltaOctaChrome0122IndSize = 955; // Largest subscript +extern const uint32 kDeltaOctaIndSize = kDeltaOctaChrome0122IndSize; // Source-agnostic named constant static const uint32 kDeltaOctaChrome0122Ind[kDeltaOctaChrome0122IndSize] = { // [0000] 0x00000000, 0x00000000, 0x00001324, 0x00000e1c, // -- -- bh.un.un_900 is.un.un_800 @@ -4580,6 +4581,7 @@ static const IndirectProbBucket4 kDeltaOctaChrome0122_2[kDeltaOctaChrome0122_2Si static const uint32 kDeltaOctaChrome0122_2SizeOne = 2; // One-langprob count static const uint32 kDeltaOctaChrome0122_2IndSize = 2; // Largest subscript +extern const uint32 kDeltaOcta2IndSize = kDeltaOctaChrome0122_2IndSize; // Source-agnostic named constant static const uint32 kDeltaOctaChrome0122_2Ind[kDeltaOctaChrome0122_2IndSize] = { // [0000] --- double_langprob_start=0002 --- 0x00000000, 0x00000000, // -- -- diff --git a/internal/cld2_generated_distinctocta0122.cc b/internal/cld2_generated_distinctocta0122.cc index 1d2d896..1f3627c 100644 --- a/internal/cld2_generated_distinctocta0122.cc +++ b/internal/cld2_generated_distinctocta0122.cc @@ -8284,8 +8284,10 @@ static const IndirectProbBucket4 kDistinctOcta0122[kDistinctOcta0122Size] = { }; // table_hash = 981b-55ce, unused_entries = 14207 (43.36%) -static const uint32 kDistinctOcta0122SizeOne = 98; // Bucket count one-lang -static const uint32 kDistinctOcta0122Ind[98] = { +static const uint32 kDistinctOcta0122SizeOne = 98; // Bucket count one-lang +static const uint32 kDistinctOcta0122IndSize = 98; // Largest subscript +extern const uint32 kDistinctOctaIndSize = kDistinctOcta0122IndSize; // Source-agnostic named constant +static const uint32 kDistinctOcta0122Ind[kDistinctOcta0122IndSize] = { // [0000] 0x00000000, 0x00000000, 0x00001706, 0x00002a06, // -- -- sr.un.un_400 bo.un.un_400 0x00003406, 0x00003303, 0x00003906, 0x00001e01, // xh.un.un_400 nn.un.un_300 dz.un.un_400 ms.un.un_200 @@ -8341,8 +8343,10 @@ static const IndirectProbBucket4 kDistinctOcta0122_2[kDistinctOcta0122_2Size] = }; // table_hash = ffff-ffff, unused_entries = 0 (0.00%) -static const uint32 kDistinctOcta0122_2SizeOne = 2; // Bucket count one-lang -static const uint32 kDistinctOcta0122_2Ind[2] = { +static const uint32 kDistinctOcta0122_2SizeOne = 2; // Bucket count one-lang +static const uint32 kDistinctOcta0122_2IndSize = 2; // Largest subscript +extern const uint32 kDistinctOcta2IndSize = kDistinctOcta0122_2IndSize; // Source-agnostic named constant +static const uint32 kDistinctOcta0122_2Ind[kDistinctOcta0122_2IndSize] = { // [0000] --- double_langprob_start=0002 --- 0x00000000, 0x00000000, // -- -- // diff --git a/internal/cld2_generated_distinctoctachrome0122.cc b/internal/cld2_generated_distinctoctachrome0122.cc index e8171ca..7485bfb 100644 --- a/internal/cld2_generated_distinctoctachrome0122.cc +++ b/internal/cld2_generated_distinctoctachrome0122.cc @@ -2137,6 +2137,7 @@ static const IndirectProbBucket4 kDistinctOctaChrome0122[kDistinctOctaChrome0122 static const uint32 kDistinctOctaChrome0122SizeOne = 76; // One-langprob count static const uint32 kDistinctOctaChrome0122IndSize = 76; // Largest subscript +extern const uint32 kDistinctOctaIndSize = kDistinctOctaChrome0122IndSize; // Source-agnostic named constant static const uint32 kDistinctOctaChrome0122Ind[kDistinctOctaChrome0122IndSize] = { // [0000] 0x00000000, 0x00000000, 0x00000b03, 0x00001c03, // -- -- es.un.un_300 id.un.un_300 @@ -2187,6 +2188,7 @@ static const IndirectProbBucket4 kDistinctOctaChrome0122_2[kDistinctOctaChrome01 static const uint32 kDistinctOctaChrome0122_2SizeOne = 2; // One-langprob count static const uint32 kDistinctOctaChrome0122_2IndSize = 2; // Largest subscript +extern const uint32 kDistinctOcta2IndSize = kDistinctOctaChrome0122_2IndSize; // Source-agnostic named constant static const uint32 kDistinctOctaChrome0122_2Ind[kDistinctOctaChrome0122_2IndSize] = { // [0000] --- double_langprob_start=0002 --- 0x00000000, 0x00000000, // -- -- diff --git a/internal/cld2_generated_quadchrome0122_16.cc b/internal/cld2_generated_quadchrome0122_16.cc index 45018c8..1ae34ef 100644 --- a/internal/cld2_generated_quadchrome0122_16.cc +++ b/internal/cld2_generated_quadchrome0122_16.cc @@ -32952,6 +32952,7 @@ static const IndirectProbBucket4 kQuadChrome0122_16[kQuadChrome0122_16Size] = { static const uint32 kQuadChrome0122_16SizeOne = 28727; // One-langprob count static const uint32 kQuadChrome0122_16IndSize = 28727; // Largest subscript +extern const uint32 kQuadChromeIndSize = kQuadChrome0122_16IndSize; // Source-agnostic named constant static const uint32 kQuadChrome0122_16Ind[kQuadChrome0122_16IndSize] = { // [0000] 0x00000000, 0x00000000, 0x08040709, 0x00000a24, // -- -- bg.ru.uk_444 mk.un.un_900 @@ -50184,6 +50185,7 @@ static const IndirectProbBucket4 kQuadChrome0122_16_2[kQuadChrome0122_16_2Size] static const uint32 kQuadChrome0122_16_2SizeOne = 8108; // One-langprob count static const uint32 kQuadChrome0122_16_2IndSize = 8108; // Largest subscript +extern const uint32 kQuadChrome2IndSize = kQuadChrome0122_16_2IndSize; // Source-agnostic named constant static const uint32 kQuadChrome0122_16_2Ind[kQuadChrome0122_16_2IndSize] = { // [0000] 0x00000000, 0x00000000, 0x080c03a7, 0x20005504, // -- -- nl.sv.no_532 rw.sq.un_320 diff --git a/internal/cld2_generated_quadchrome0122_19.cc b/internal/cld2_generated_quadchrome0122_19.cc index 9c424a5..0d189ab 100644 --- a/internal/cld2_generated_quadchrome0122_19.cc +++ b/internal/cld2_generated_quadchrome0122_19.cc @@ -32952,6 +32952,7 @@ static const IndirectProbBucket4 kQuadChrome0122_19[kQuadChrome0122_19Size] = { static const uint32 kQuadChrome0122_19SizeOne = 31722; // One-langprob count static const uint32 kQuadChrome0122_19IndSize = 31722; // Largest subscript +extern const uint32 kQuadChromeIndSize = kQuadChrome0122_19IndSize; // Source-agnostic named constant static const uint32 kQuadChrome0122_19Ind[kQuadChrome0122_19IndSize] = { // [0000] 0x00000000, 0x00000000, 0x08040709, 0x00000a24, // -- -- bg.ru.uk_444 mk.un.un_900 @@ -59320,8 +59321,9 @@ static const IndirectProbBucket4 kQuadChrome0122_19_2[kQuadChrome0122_19_2Size] }; // table_hash = 8afb-a95b, unused_entries = 75 (0.04%) -static const uint32 kQuadChrome0122_19_2SizeOne = 13863; // One-langprob count +static const uint32 kQuadChrome0122_19_2SizeOne = 13863; // One-langprob count static const uint32 kQuadChrome0122_19_2IndSize = 13863; // Largest subscript +extern const uint32 kQuadChrome2IndSize = kQuadChrome0122_19_2IndSize; // Source-agnostic named constant static const uint32 kQuadChrome0122_19_2Ind[kQuadChrome0122_19_2IndSize] = { // [0000] 0x00000000, 0x00000000, 0x080c03a7, 0x20005504, // -- -- nl.sv.no_532 rw.sq.un_320 diff --git a/internal/cld2_generated_quadchrome0122_2.cc b/internal/cld2_generated_quadchrome0122_2.cc index 533b3c9..b382214 100644 --- a/internal/cld2_generated_quadchrome0122_2.cc +++ b/internal/cld2_generated_quadchrome0122_2.cc @@ -65750,6 +65750,7 @@ static const IndirectProbBucket4 kQuadChrome0122_2[kQuadChrome0122_2Size] = { static const uint32 kQuadChrome0122_2SizeOne = 50708; // One-langprob count static const uint32 kQuadChrome0122_2IndSize = 50708; // Largest subscript +extern const uint32 kQuadChromeIndSize = kQuadChrome0122_2IndSize; // Source-agnostic named constant static const uint32 kQuadChrome0122_2Ind[kQuadChrome0122_2IndSize] = { // [0000] 0x00000000, 0x00000000, 0x08040709, 0x06000c13, // -- -- bg.ru.uk_444 sv.de.un_650 @@ -81672,6 +81673,7 @@ static const IndirectProbBucket4 kQuadChrome0122_2_2[kQuadChrome0122_2_2Size] = static const uint32 kQuadChrome0122_2_2SizeOne = 2; // One-langprob count static const uint32 kQuadChrome0122_2_2IndSize = 2; // Largest subscript +extern const uint32 kQuadChrome2IndSize = kQuadChrome0122_2_2IndSize; // Source-agnostic named constant static const uint32 kQuadChrome0122_2_2Ind[kQuadChrome0122_2_2IndSize] = { // [0000] --- double_langprob_start=0002 --- 0x00000000, 0x00000000, // -- -- diff --git a/internal/cld_generated_cjk_delta_bi_32.cc b/internal/cld_generated_cjk_delta_bi_32.cc index c30b5d6..58afd2a 100644 --- a/internal/cld_generated_cjk_delta_bi_32.cc +++ b/internal/cld_generated_cjk_delta_bi_32.cc @@ -8252,7 +8252,7 @@ static const IndirectProbBucket4 kCjkDeltaBi[kCjkDeltaBiSize] = { }; static const uint32 kCjkDeltaBiSizeOne = 33; // One-langprob count -static const uint32 kCjkDeltaBiIndSize = 33; // Largest subscript +extern const uint32 kCjkDeltaBiIndSize = 33; // Largest subscript static const uint32 kCjkDeltaBiInd[kCjkDeltaBiIndSize] = { // [0000] 0x00000000, 0x00001d2d, 0x00001d1c, 0x00000000, diff --git a/internal/cld_generated_cjk_delta_bi_4.cc b/internal/cld_generated_cjk_delta_bi_4.cc index 6f6d732..6b91d45 100644 --- a/internal/cld_generated_cjk_delta_bi_4.cc +++ b/internal/cld_generated_cjk_delta_bi_4.cc @@ -1102,7 +1102,7 @@ static const IndirectProbBucket4 kCjkDeltaBi[kCjkDeltaBiSize] = { // table_hash = 860b-1885, unused_entries = 1630 (39.79%) static const uint32 kCjkDeltaBiSizeOne = 34; // One-langprob count -static const uint32 kCjkDeltaBiIndSize = 34; // Largest subscript +extern const uint32 kCjkDeltaBiIndSize = 34; // Largest subscript static const uint32 kCjkDeltaBiInd[kCjkDeltaBiIndSize] = { // [0000] 0x00000000, 0x00000000, 0x00001d1c, 0x00000242, // -- -- zh-Hant.un.un_800 ja.un.un_C00 diff --git a/internal/cld_generated_score_quad_octa_0122.cc b/internal/cld_generated_score_quad_octa_0122.cc index ea3bb06..7e6b81a 100644 --- a/internal/cld_generated_score_quad_octa_0122.cc +++ b/internal/cld_generated_score_quad_octa_0122.cc @@ -25,7 +25,7 @@ namespace CLD2 { // Average score per 1024 bytes -static const int kAvgDeltaOctaScoreSize = 614 * 4; +extern const int kAvgDeltaOctaScoreSize = 614 * 4; extern const short kAvgDeltaOctaScore[kAvgDeltaOctaScoreSize] = { // Latn Cyrl Arab Other script // Updated 20140204 for CLD2 full diff --git a/internal/cld_generated_score_quad_octa_0122_2.cc b/internal/cld_generated_score_quad_octa_0122_2.cc index 1798697..2eb5316 100644 --- a/internal/cld_generated_score_quad_octa_0122_2.cc +++ b/internal/cld_generated_score_quad_octa_0122_2.cc @@ -16,7 +16,7 @@ namespace CLD2 { // Average score per 1024 bytes -static const int kAvgDeltaOctaScoreSize = 614 * 4; +extern const int kAvgDeltaOctaScoreSize = 614 * 4; extern const short kAvgDeltaOctaScore[kAvgDeltaOctaScoreSize] = { // Latn Cyrl Arab Other script // Updated 20140202 for CLD2 Chrome 256K entries diff --git a/internal/generated_distinct_bi_0.cc b/internal/generated_distinct_bi_0.cc index e9e763a..b6c5744 100644 --- a/internal/generated_distinct_bi_0.cc +++ b/internal/generated_distinct_bi_0.cc @@ -32,7 +32,7 @@ static const IndirectProbBucket4 kDistinctBiTable[kDistinctBiTableSize] = { }; static const uint32 kDistinctBiTableSizeOne = 1; // One-langprob count -static const uint32 kDistinctBiTableIndSize = 1; // Largest subscript +extern const uint32 kDistinctBiTableIndSize = 1; // Largest subscript static const uint32 kDistinctBiTableInd[kDistinctBiTableIndSize] = { // [0000] 0x00000000, };