Amebis/cld2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enable Dynamic Mode for CLD2. See issue 6 for more information on dynamic mode:

Browse Source 
https://code.google.com/p/cld2/issues/detail?id=6


git-svn-id: https://cld2.googlecode.com/svn/trunk@151 b252ecd4-b096-bf77-eb8e-91563289f87e
This commit is contained in:
andrewhayden@google.com
2014-03-03 15:20:05 +00:00
parent 0fb71b3bda
commit cffbd73e13
12 changed files with 1541 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
internal/compact_lang_det_test.cc
View File

@@ -42,16 +42,18 @@ typedef int32 Encoding;
static const Encoding UNKNOWN_ENCODING = 0;
// Linker supplies the right tables; see ScoringTables compact_lang_det_impl.cc
// These are here JUST for printing versions
extern const UTF8PropObj cld_generated_CjkUni_obj;
extern const CLD2TableSummary kCjkDeltaBi_obj;
extern const CLD2TableSummary kDistinctBiTable_obj;
extern const CLD2TableSummary kQuad_obj;
extern const CLD2TableSummary kDeltaOcta_obj;
extern const CLD2TableSummary kDistinctOcta_obj;
extern const CLD2TableSummary kOcta2_obj;
extern const short kAvgDeltaOctaScore[];
#ifndef CLD2_DYNAMIC_MODE
// Linker supplies the right tables; see ScoringTables compact_lang_det_impl.cc
// These are here JUST for printing versions
extern const UTF8PropObj cld_generated_CjkUni_obj;
extern const CLD2TableSummary kCjkDeltaBi_obj;
extern const CLD2TableSummary kDistinctBiTable_obj;
extern const CLD2TableSummary kQuad_obj;
extern const CLD2TableSummary kDeltaOcta_obj;
extern const CLD2TableSummary kDistinctOcta_obj;
extern const CLD2TableSummary kOcta2_obj;
extern const short kAvgDeltaOctaScore[];
#endif
bool FLAGS_cld_version = false;
bool FLAGS_cld_html = true;
@@ -201,6 +203,7 @@ void DumpLanguages(Language summary_lang,
int main(int argc, char** argv) {
if (FLAGS_cld_version) {
#ifndef CLD2_DYNAMIC_MODE
printf("%s %4dKB uni build date, bytes\n",
"........",
cld_generated_CjkUni_obj.total_size >> 10);
@@ -216,11 +219,14 @@ int main(int argc, char** argv) {
kDeltaOcta_obj.kCLDTableBuildDate,
(kDeltaOcta_obj.kCLDTableSize *
sizeof(IndirectProbBucket4)) >> 10);
#else
printf("FLAGS_cld_version doesn't work with dynamic data mode\n");
#endif
exit(0);
} // End FLAGS_cld_version
int flags = 0;
bool get_vector = false;
const char* data_file = NULL;
const char* fname = NULL;
for (int i = 1; i < argc; ++i) {
if (argv[i][0] != '-') {fname = argv[i];}
@@ -230,8 +236,19 @@ int main(int argc, char** argv) {
if (strcmp(argv[i], "--verbose") == 0) {flags |= kCLDFlagVerbose;}
if (strcmp(argv[i], "--echo") == 0) {flags |= kCLDFlagEcho;}
if (strcmp(argv[i], "--vector") == 0) {get_vector = true;}
if (strcmp(argv[i], "--data-file") == 0) { data_file = argv[++i];}
}
#ifdef CLD2_DYNAMIC_MODE
if (data_file == NULL) {
fprintf(stderr, "When running in dynamic mode, you must specify --data-file [FILE]\n");
return -1;
}
fprintf(stdout, "Loading data from: %s\n", data_file);
CLD2::loadData(data_file);
fprintf(stdout, "Data loaded, test commencing\n");
#endif
FILE* fin;
if (fname == NULL) {
fin = stdin;