Amebis/cld2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove the old 0715, 0122 and 0604 data sets for Chrome.

Browse Source 
Adjust scripts accordingly and eliminate old unittest file as well.
As of this revision, the default for compilation in all the scripts
EXCEPT for compile_full.sh is the 20141015 dataset. The compile_full.sh
script still builds the most recent pre-20141015 data set, and should
also be updated soon.

The suffix (0122, 0715, 0604, or 20141015) has simply been deleted from
all affected filenames, e.g.:
  cld2_generated_deltaoctachrome0122.cc -> cld2_generated_deltaoctachrome.cc

Moving forward the intention is not keep the old data files around. Repository
history can be used to check out old versions of data files as necessary.

The clean.sh script has also been updated with the names of the new executables
produced in the compilation and testing process along with the svn:ignore
propset.

In the next commit, the compile20141015 will become compile.sh.



git-svn-id: https://cld2.googlecode.com/svn/trunk@190 b252ecd4-b096-bf77-eb8e-91563289f87e
This commit is contained in:
andrewhayden@google.com
2014-11-06 22:49:10 +00:00
parent 077be12beb
commit b2c2d34ea7
21 changed files with 89 additions and 294433 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
internal/cld2_generated_deltaoctachrome20141015.cc → internal/cld2_generated_deltaoctachrome.cc
View File

4621
internal/cld2_generated_deltaoctachrome0122.cc
View File

File diff suppressed because it is too large Load Diff

4547
internal/cld2_generated_deltaoctachrome0614.cc
View File

File diff suppressed because it is too large Load Diff

0
internal/cld2_generated_distinctoctachrome20141015.cc → internal/cld2_generated_distinctoctachrome.cc
View File

2228
internal/cld2_generated_distinctoctachrome0122.cc
View File

File diff suppressed because it is too large Load Diff

2188
internal/cld2_generated_distinctoctachrome0604.cc
View File

File diff suppressed because it is too large Load Diff

52761
internal/cld2_generated_quadchrome0122_16.cc
View File

File diff suppressed because it is too large Load Diff

63704
internal/cld2_generated_quadchrome0122_19.cc
View File

File diff suppressed because it is too large Load Diff

81713
internal/cld2_generated_quadchrome0122_2.cc
View File

File diff suppressed because it is too large Load Diff

82295
internal/cld2_generated_quadchrome0715.cc
View File

File diff suppressed because it is too large Load Diff

0
internal/cld2_generated_quadchrome20141015_16.cc → internal/cld2_generated_quadchrome_16.cc
View File

0
internal/cld2_generated_quadchrome20141015_2.cc → internal/cld2_generated_quadchrome_2.cc
View File

58
internal/cld2_unittest.cc
View File

@@ -145,26 +145,46 @@ static const TestPair kTestPair[] = {
{YIDDISH, kTeststr_yi_Hebr},
// Added 2013.08.31 so-Latn ig-Latn ha-Latn yo-Latn zu-Latn
{SOMALI, kTeststr_so_Latn},
{IGBO, kTeststr_ig_Latn},
{HAUSA, kTeststr_ha_Latn},
{YORUBA, kTeststr_yo_Latn},
{ZULU, kTeststr_zu_Latn},
// Deleted 2014.10.15 so-Latn ig-Latn ha-Latn yo-Latn zu-Latn
//{SOMALI, kTeststr_so_Latn},
//{IGBO, kTeststr_ig_Latn},
//{HAUSA, kTeststr_ha_Latn},
//{YORUBA, kTeststr_yo_Latn},
//{ZULU, kTeststr_zu_Latn},
// Added 2014.01.22 bs-Latn
{BOSNIAN, kTeststr_bs_Latn},
// 2 statistically-close languages
// Added 2014.10.15
{KAZAKH, kTeststr_kk_Cyrl},
{KURDISH, kTeststr_ku_Latn}, // aka kmr
{KYRGYZ, kTeststr_ky_Cyrl},
{MALAGASY, kTeststr_mg_Latn},
{MALAYALAM, kTeststr_ml_Mlym},
{BURMESE, kTeststr_my_Mymr},
{NYANJA, kTeststr_ny_Latn},
{SINHALESE, kTeststr_si_Sinh}, // aka SINHALA
{SESOTHO, kTeststr_st_Latn},
{SUNDANESE, kTeststr_su_Latn},
{TAJIK, kTeststr_tg_Cyrl},
{UZBEK, kTeststr_uz_Latn},
{UZBEK, kTeststr_uz_Cyrl},
// 2 statistically-close languages
{INDONESIAN, kTeststr_id_close},
{MALAY, kTeststr_ms_close},
// Simple intermixed French/English text
{FRENCH, kTeststr_fr_en_Latn},
// Simple English with bad UTF-8
{UNKNOWN_LANGUAGE, kTeststr_en_Latn_bad_UTF8},
// Cross-check the main quadgram table build date
// Change the expected language each time it is rebuilt
//{WELSH, kTeststr_version}, // 2013.07.15
{AZERBAIJANI, kTeststr_version}, // 2014.01.31
// {WELSH, kTeststr_version}, // 2013.07.15
// {AZERBAIJANI, kTeststr_version}, // 2014.01.31
{TURKISH, kTeststr_version}, // 2014.10.16
{UNKNOWN_LANGUAGE, NULL}, // Must be last
};
@@ -183,8 +203,9 @@ bool OneTest(int flags, bool get_vector,
ResultChunkVector resultchunkvector;
int text_bytes;
bool is_reliable;
int valid_prefix_bytes;
Language lang_detected = ExtDetectLanguageSummary(
Language lang_detected = ExtDetectLanguageSummaryCheckUTF8(
buffer,
buffer_length,
is_plain_text,
@@ -195,10 +216,17 @@ bool OneTest(int flags, bool get_vector,
normalized_score3,
get_vector ? &resultchunkvector : NULL,
&text_bytes,
&is_reliable);
&is_reliable,
&valid_prefix_bytes);
// expose DumpExtLang DumpLanguages
bool good_utf8 = (valid_prefix_bytes == buffer_length);
if (!good_utf8) {
fprintf(stderr, "*** Bad UTF-8 after %d bytes<br>\n", valid_prefix_bytes);
fprintf(stdout, "*** Bad UTF-8 after %d bytes\n", valid_prefix_bytes);
}
bool ok = (lang_detected == lang_expected);
ok &= good_utf8;
if (!ok) {
if ((flags & kCLDFlagHtml) != 0) {
@@ -305,6 +333,10 @@ int RunTests (int flags, bool get_vector) {
const char* buffer = kTestPair[i].text;
int buffer_length = strlen(buffer);
bool ok = OneTest(flags, get_vector, lang_expected, buffer, buffer_length);
if (kTestPair[i].text == kTeststr_en_Latn_bad_UTF8) {
// We expect this one to fail, so flip the value of ok
ok = !ok;
}
any_fail |= (!ok);
++i;
}
@@ -350,6 +382,10 @@ int RunTests (int flags, bool get_vector) {
const char* buffer = kTestPair[i].text;
int buffer_length = strlen(buffer);
bool ok = OneTest(flags, get_vector, lang_expected, buffer, buffer_length);
if (kTestPair[i].text == kTeststr_en_Latn_bad_UTF8) {
// We expect this one to fail, so flip the value of ok
ok = !ok;
}
any_fail |= (!ok);
++i;
}

329
internal/cld2_unittest_20141015.cc
View File

@@ -1,329 +0,0 @@
// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: dsites@google.com (Dick Sites)
//
// Unit test compact language detector, CLD2
// Compile with -Davoid_utf8_string_constants if your compiler cannot
// handle UTF-8 string constants
//
#include <stdio.h>
#include <string.h>
#include "../public/compact_lang_det.h"
#include "../public/encodings.h"
#include "unittest_data.h"
namespace CLD2 {
// Test strings.
const char* kTeststr_en =
"confiscation of goods is assigned as the penalty part most of the courts "
"consist of members and when it is necessary to bring public cases before a "
"jury of members two courts combine for the purpose the most important cases "
"of all are brought jurors or";
typedef struct {
Language lang;
const char* text;
} TestPair;
static const TestPair kTestPair[] = {
// A simple case to begin
{ENGLISH, kTeststr_en},
// 20 languages recognized via Unicode script
{ARMENIAN, kTeststr_hy_Armn},
{CHEROKEE, kTeststr_chr_Cher},
{DHIVEHI, kTeststr_dv_Thaa},
{GEORGIAN, kTeststr_ka_Geor},
{GREEK, kTeststr_el_Grek},
{GUJARATI, kTeststr_gu_Gujr},
{INUKTITUT, kTeststr_iu_Cans},
{KANNADA, kTeststr_kn_Knda},
{KHMER, kTeststr_km_Khmr},
{LAOTHIAN, kTeststr_lo_Laoo},
{LIMBU, kTeststr_lif_Limb},
{MALAYALAM, kTeststr_ml_Mlym},
{ORIYA, kTeststr_or_Orya},
{PUNJABI, kTeststr_pa_Guru},
{SINHALESE, kTeststr_si_Sinh},
{SYRIAC, kTeststr_syr_Syrc},
{TAGALOG, kTeststr_tl_Tglg}, // Also in quadgram list below
{TAMIL, kTeststr_ta_Taml},
{TELUGU, kTeststr_te_Telu},
{THAI, kTeststr_th_Thai},
// 4 languages regognized via single letters
{CHINESE, kTeststr_zh_Hans},
{CHINESE_T, kTeststr_zh_Hant},
{JAPANESE, kTeststr_ja_Hani},
{KOREAN, kTeststr_ko_Hani},
// 60 languages recognized via combinations of four letters
{AFRIKAANS, kTeststr_af_Latn},
{ALBANIAN, kTeststr_sq_Latn},
{ARABIC, kTeststr_ar_Arab},
{AZERBAIJANI, kTeststr_az_Latn},
{BASQUE, kTeststr_eu_Latn},
{BELARUSIAN, kTeststr_be_Cyrl},
{BENGALI, kTeststr_bn_Beng}, // No Assamese in subset
{BIHARI, kTeststr_bh_Deva},
{BULGARIAN, kTeststr_bg_Cyrl},
{CATALAN, kTeststr_ca_Latn},
{CEBUANO, kTeststr_ceb_Latn},
{CROATIAN, kTeststr_hr_Latn},
{CZECH, kTeststr_cs_Latn},
{DANISH, kTeststr_da_Latn},
{DUTCH, kTeststr_nl_Latn},
{ENGLISH, kTeststr_en_Latn},
{ESTONIAN, kTeststr_et_Latn},
{FINNISH, kTeststr_fi_Latn},
{FRENCH, kTeststr_fr_Latn},
{GALICIAN, kTeststr_gl_Latn},
{GANDA, kTeststr_lg_Latn},
{GERMAN, kTeststr_de_Latn},
{HAITIAN_CREOLE, kTeststr_ht_Latn},
{HEBREW, kTeststr_iw_Hebr},
{HINDI, kTeststr_hi_Deva},
{HMONG, kTeststr_blu_Latn},
{HUNGARIAN, kTeststr_hu_Latn},
{ICELANDIC, kTeststr_is_Latn},
{INDONESIAN, kTeststr_id_Latn},
{IRISH, kTeststr_ga_Latn},
{ITALIAN, kTeststr_it_Latn},
{JAVANESE, kTeststr_jw_Latn},
{KINYARWANDA, kTeststr_rw_Latn},
{LATVIAN, kTeststr_lv_Latn},
{LITHUANIAN, kTeststr_lt_Latn},
{MACEDONIAN, kTeststr_mk_Cyrl},
{MALAY, kTeststr_ms_Latn},
{MALTESE, kTeststr_mt_Latn},
{MARATHI, kTeststr_mr_Deva},
{NEPALI, kTeststr_ne_Deva},
{NORWEGIAN, kTeststr_no_Latn},
{PERSIAN, kTeststr_fa_Arab},
{POLISH, kTeststr_pl_Latn},
{PORTUGUESE, kTeststr_pt_Latn},
{ROMANIAN, kTeststr_ro_Latn},
{ROMANIAN, kTeststr_ro_Cyrl},
{RUSSIAN, kTeststr_ru_Cyrl},
{SCOTS_GAELIC, kTeststr_gd_Latn},
{SERBIAN, kTeststr_sr_Cyrl},
{SERBIAN, kTeststr_sr_Latn},
{SLOVAK, kTeststr_sk_Latn},
{SLOVENIAN, kTeststr_sl_Latn},
{SPANISH, kTeststr_es_Latn},
{SWAHILI, kTeststr_sw_Latn},
{SWEDISH, kTeststr_sv_Latn},
{TAGALOG, kTeststr_tl_Latn},
{TURKISH, kTeststr_tr_Latn},
{UKRAINIAN, kTeststr_uk_Cyrl},
{URDU, kTeststr_ur_Arab},
{VIETNAMESE, kTeststr_vi_Latn},
{WELSH, kTeststr_cy_Latn},
{YIDDISH, kTeststr_yi_Hebr},
// Added 2013.08.31 so-Latn ig-Latn ha-Latn yo-Latn zu-Latn
// Deleted 2014.10.15 so-Latn ig-Latn ha-Latn yo-Latn zu-Latn
//{SOMALI, kTeststr_so_Latn},
//{IGBO, kTeststr_ig_Latn},
//{HAUSA, kTeststr_ha_Latn},
//{YORUBA, kTeststr_yo_Latn},
//{ZULU, kTeststr_zu_Latn},
// Added 2014.01.22 bs-Latn
{BOSNIAN, kTeststr_bs_Latn},
// Added 2014.10.15
{KAZAKH, kTeststr_kk_Cyrl},
{KURDISH, kTeststr_ku_Latn}, // aka kmr
{KYRGYZ, kTeststr_ky_Cyrl},
{MALAGASY, kTeststr_mg_Latn},
{MALAYALAM, kTeststr_ml_Mlym},
{BURMESE, kTeststr_my_Mymr},
{NYANJA, kTeststr_ny_Latn},
{SINHALESE, kTeststr_si_Sinh}, // aka SINHALA
{SESOTHO, kTeststr_st_Latn},
{SUNDANESE, kTeststr_su_Latn},
{TAJIK, kTeststr_tg_Cyrl},
{UZBEK, kTeststr_uz_Latn},
{UZBEK, kTeststr_uz_Cyrl},
// 2 statistically-close languages
{INDONESIAN, kTeststr_id_close},
{MALAY, kTeststr_ms_close},
// Simple intermixed French/English text
{FRENCH, kTeststr_fr_en_Latn},
// Simple English with bad UTF-8
{UNKNOWN_LANGUAGE, kTeststr_en_Latn_bad_UTF8},
// Cross-check the main quadgram table build date
// Change the expected language each time it is rebuilt
// {WELSH, kTeststr_version}, // 2013.07.15
// {AZERBAIJANI, kTeststr_version}, // 2014.01.31
{TURKISH, kTeststr_version}, // 2014.10.16
{UNKNOWN_LANGUAGE, NULL}, // Must be last
};
bool OneTest(int flags, bool get_vector,
Language lang_expected, const char* buffer, int buffer_length) {
bool is_plain_text = true;
const char* tldhint = "";
const Encoding enchint = UNKNOWN_ENCODING;
const Language langhint = UNKNOWN_LANGUAGE;
const CLDHints cldhints = {NULL, tldhint, enchint, langhint};
Language language3[3];
int percent3[3];
double normalized_score3[3];
ResultChunkVector resultchunkvector;
int text_bytes;
bool is_reliable;
int valid_prefix_bytes;
Language lang_detected = ExtDetectLanguageSummaryCheckUTF8(
buffer,
buffer_length,
is_plain_text,
&cldhints,
flags,
language3,
percent3,
normalized_score3,
get_vector ? &resultchunkvector : NULL,
&text_bytes,
&is_reliable,
&valid_prefix_bytes);
// expose DumpExtLang DumpLanguages
bool good_utf8 = (valid_prefix_bytes == buffer_length);
if (!good_utf8) {
fprintf(stderr, "*** Bad UTF-8 after %d bytes<br>\n", valid_prefix_bytes);
fprintf(stdout, "*** Bad UTF-8 after %d bytes\n", valid_prefix_bytes);
}
bool ok = (lang_detected == lang_expected);
ok &= good_utf8;
if (!ok) {
if ((flags & kCLDFlagHtml) != 0) {
fprintf(stderr, "*** Wrong result. expected %s, detected %s<br>\n",
LanguageName(lang_expected), LanguageName(lang_detected));
}
fprintf(stdout, "*** Wrong result. expected %s, detected %s\n",
LanguageName(lang_expected), LanguageName(lang_detected));
fprintf(stdout, "%s\n\n", buffer);
}
if (get_vector) {
DumpResultChunkVector(stderr, buffer, &resultchunkvector);
}
#if 0
DumpExtLang(flags, summary_lang, language3, percent3, normalized_score3,
text_bytes, is_reliable, n);
if ((flags & kCLDFlagHtml) != 0) {
DumpLanguages(summary_lang,
language3, percent3, text_bytes, is_reliable, n);
}
fprintf(stdout, " SummaryLanguage %s%s at %u of %d, %s\n",
LanguageName(summary_lang),
is_reliable ? "" : "(un-reliable)",
bytes_consumed,
n,
argv[1]);
#endif
return ok;
}
void InitHtmlOut(int flags) {
#if 1
if ((flags & kCLDFlagHtml) != 0) {
// Begin HTML file
fprintf(stderr, "<html><meta charset=\"UTF-8\"><body>\n");
// Encourage browsers to print background colors
fprintf(stderr, "<style media=\"print\" type=\"text/css\"> "
":root { -webkit-print-color-adjust: exact; } </style>\n");
fprintf(stderr, "<span style=\"font-size: 7pt\">\n");
fprintf(stderr, "file = %s<br>\n", "cld2_unittest");
}
#endif
}
void FinishHtmlOut(int flags) {
#if 1
if ((flags & kCLDFlagHtml) != 0) {
fprintf(stderr, "\n</span></body></html>\n");
}
#endif
}
int RunTests (int flags, bool get_vector) {
fprintf(stdout, "CLD2 version: %s\n", CLD2::DetectLanguageVersion());
InitHtmlOut(flags);
bool any_fail = false;
int i = 0;
while (kTestPair[i].text != NULL) {
Language lang_expected = kTestPair[i].lang;
const char* buffer = kTestPair[i].text;
int buffer_length = strlen(buffer);
bool ok = OneTest(flags, get_vector, lang_expected, buffer, buffer_length);
if (kTestPair[i].text == kTeststr_en_Latn_bad_UTF8) {
// We expect this one to fail, so flip the value of ok
ok = !ok;
}
any_fail |= (!ok);
++i;
}
if (any_fail) {
fprintf(stderr, "FAIL\n");
fprintf(stdout, "FAIL\n");
} else {
fprintf(stderr, "PASS\n");
fprintf(stdout, "PASS\n");
}
FinishHtmlOut(flags);
return 0;
}
} // End namespace CLD2
int main(int argc, char** argv) {
// Get command-line flags
int flags = 0;
bool get_vector = false;
for (int i = 1; i < argc; ++i) {
if (strcmp(argv[i], "--html") == 0) {flags |= CLD2::kCLDFlagHtml;}
if (strcmp(argv[i], "--cr") == 0) {flags |= CLD2::kCLDFlagCr;}
if (strcmp(argv[i], "--verbose") == 0) {flags |= CLD2::kCLDFlagVerbose;}
if (strcmp(argv[i], "--quiet") == 0) {flags |= CLD2::kCLDFlagQuiet;}
if (strcmp(argv[i], "--echo") == 0) {flags |= CLD2::kCLDFlagEcho;}
if (strcmp(argv[i], "--vector") == 0) {get_vector = true;}
}
return CLD2::RunTests(flags, get_vector);
}

0
internal/cld_generated_score_quad_octa_20141015_2.cc → internal/cld_generated_score_quad_octa_2.cc
View File

4
internal/clean.sh
View File

@@ -27,3 +27,7 @@ rm -f cld2_data.bin
rm -f compact_lang_det_dynamic_test_chrome
rm -f cld2_dynamic_unittest
rm -f libcld2_dynamic.so
rm -f compact_lang_det_test_chrome_16
rm -f compact_lang_det_test_chrome_2
rm -f cld2_unittest_avoid_chrome_2
rm -f cld2_unittest_chrome_2

12
internal/compile.sh
View File

@@ -22,8 +22,8 @@ g++ -O2 -m64 compact_lang_det_test.cc \
tote.cc utf8statetable.cc \
cld_generated_cjk_uni_prop_80.cc cld2_generated_cjk_compatible.cc \
cld_generated_cjk_delta_bi_4.cc generated_distinct_bi_0.cc \
cld2_generated_quadchrome0122_2.cc cld2_generated_deltaoctachrome0122.cc \
cld2_generated_distinctoctachrome0122.cc cld_generated_score_quad_octa_0122_2.cc \
cld2_generated_quadchrome_2.cc cld2_generated_deltaoctachrome.cc \
cld2_generated_distinctoctachrome.cc cld_generated_score_quad_octa_2.cc \
-o compact_lang_det_test_chrome
echo " compact_lang_det_test_chrome compiled"
@@ -35,8 +35,8 @@ g++ -O2 -m64 cld2_unittest.cc \
tote.cc utf8statetable.cc \
cld_generated_cjk_uni_prop_80.cc cld2_generated_cjk_compatible.cc \
cld_generated_cjk_delta_bi_4.cc generated_distinct_bi_0.cc \
cld2_generated_quadchrome0122_2.cc cld2_generated_deltaoctachrome0122.cc \
cld2_generated_distinctoctachrome0122.cc cld_generated_score_quad_octa_0122_2.cc \
cld2_generated_quadchrome_2.cc cld2_generated_deltaoctachrome.cc \
cld2_generated_distinctoctachrome.cc cld_generated_score_quad_octa_2.cc \
-o cld2_unittest
echo " cld2_unittest compiled"
@@ -48,8 +48,8 @@ g++ -O2 -m64 -Davoid_utf8_string_constants cld2_unittest.cc \
tote.cc utf8statetable.cc \
cld_generated_cjk_uni_prop_80.cc cld2_generated_cjk_compatible.cc \
cld_generated_cjk_delta_bi_4.cc generated_distinct_bi_0.cc \
cld2_generated_quadchrome0122_2.cc cld2_generated_deltaoctachrome0122.cc \
cld2_generated_distinctoctachrome0122.cc cld_generated_score_quad_octa_0122_2.cc \
cld2_generated_quadchrome_2.cc cld2_generated_deltaoctachrome.cc \
cld2_generated_distinctoctachrome.cc cld_generated_score_quad_octa_2.cc \
-o cld2_unittest_avoid
echo " cld2_unittest_avoid compiled"

36
internal/compile20141015.sh
View File

@@ -22,10 +22,10 @@ g++ -O2 -m64 compact_lang_det_test.cc \
tote.cc utf8statetable.cc \
cld_generated_cjk_uni_prop_80.cc cld2_generated_cjk_compatible.cc \
cld_generated_cjk_delta_bi_4.cc generated_distinct_bi_0.cc \
cld2_generated_quadchrome20141015_2.cc cld2_generated_deltaoctachrome20141015.cc \
cld2_generated_distinctoctachrome20141015.cc cld_generated_score_quad_octa_20141015_2.cc \
-o compact_lang_det_test_chrome20141015_2
echo " compact_lang_det_test_chrome20141015_2 compiled"
cld2_generated_quadchrome_2.cc cld2_generated_deltaoctachrome.cc \
cld2_generated_distinctoctachrome.cc cld_generated_score_quad_octa_2.cc \
-o compact_lang_det_test_chrome_2
echo " compact_lang_det_test_chrome_2 compiled"
g++ -O2 -m64 compact_lang_det_test.cc \
cldutil.cc cldutil_shared.cc compact_lang_det.cc compact_lang_det_hint_code.cc \
@@ -35,13 +35,13 @@ g++ -O2 -m64 compact_lang_det_test.cc \
tote.cc utf8statetable.cc \
cld_generated_cjk_uni_prop_80.cc cld2_generated_cjk_compatible.cc \
cld_generated_cjk_delta_bi_4.cc generated_distinct_bi_0.cc \
cld2_generated_quadchrome20141015_16.cc cld2_generated_deltaoctachrome20141015.cc \
cld2_generated_distinctoctachrome20141015.cc cld_generated_score_quad_octa_20141015_2.cc \
-o compact_lang_det_test_chrome20141015_16
echo " compact_lang_det_test_chrome20141015_16 compiled"
cld2_generated_quadchrome_16.cc cld2_generated_deltaoctachrome.cc \
cld2_generated_distinctoctachrome.cc cld_generated_score_quad_octa_2.cc \
-o compact_lang_det_test_chrome_16
echo " compact_lang_det_test_chrome_16 compiled"
g++ -O2 -m64 cld2_unittest_20141015.cc \
g++ -O2 -m64 cld2_unittest.cc \
cldutil.cc cldutil_shared.cc compact_lang_det.cc compact_lang_det_hint_code.cc \
compact_lang_det_impl.cc debug.cc fixunicodevalue.cc \
generated_entities.cc generated_language.cc generated_ulscript.cc \
@@ -49,12 +49,12 @@ g++ -O2 -m64 cld2_unittest_20141015.cc \
tote.cc utf8statetable.cc \
cld_generated_cjk_uni_prop_80.cc cld2_generated_cjk_compatible.cc \
cld_generated_cjk_delta_bi_4.cc generated_distinct_bi_0.cc \
cld2_generated_quadchrome20141015_2.cc cld2_generated_deltaoctachrome20141015.cc \
cld2_generated_distinctoctachrome20141015.cc cld_generated_score_quad_octa_20141015_2.cc \
-o cld2_unittest_chrome20141015_2
echo " cld2_unittest_chrome20141015_2 compiled"
cld2_generated_quadchrome_2.cc cld2_generated_deltaoctachrome.cc \
cld2_generated_distinctoctachrome.cc cld_generated_score_quad_octa_2.cc \
-o cld2_unittest_chrome_2
echo " cld2_unittest_chrome_2 compiled"
g++ -O2 -m64 -Davoid_utf8_string_constants cld2_unittest_20141015.cc \
g++ -O2 -m64 -Davoid_utf8_string_constants cld2_unittest.cc \
cldutil.cc cldutil_shared.cc compact_lang_det.cc compact_lang_det_hint_code.cc \
compact_lang_det_impl.cc debug.cc fixunicodevalue.cc \
generated_entities.cc generated_language.cc generated_ulscript.cc \
@@ -62,10 +62,10 @@ g++ -O2 -m64 -Davoid_utf8_string_constants cld2_unittest_20141015.cc \
tote.cc utf8statetable.cc \
cld_generated_cjk_uni_prop_80.cc cld2_generated_cjk_compatible.cc \
cld_generated_cjk_delta_bi_4.cc generated_distinct_bi_0.cc \
cld2_generated_quadchrome20141015_2.cc cld2_generated_deltaoctachrome20141015.cc \
cld2_generated_distinctoctachrome20141015.cc cld_generated_score_quad_octa_20141015_2.cc \
-o cld2_unittest_avoid_chrome20141015_2
echo " cld2_unittest_avoid_chrome20141015_2 compiled"
cld2_generated_quadchrome_2.cc cld2_generated_deltaoctachrome.cc \
cld2_generated_distinctoctachrome.cc cld_generated_score_quad_octa_2.cc \
-o cld2_unittest_avoid_chrome_2
echo " cld2_unittest_avoid_chrome_2 compiled"

18
internal/compile_and_test_all.sh
View File

@@ -18,14 +18,16 @@ trap 'echo "FAILED!" && exit 1' INT TERM EXIT
# ----------------------------------------------------------------------------
echo "--> [1 of 7] Invoking: compile.sh..."
./compile.sh
./compile20141015.sh
echo "--> [2 of 7] Testing results of compile.sh..."
echo "--> compact_lang_det_test_chrome... "
echo "this is some english text" | ./compact_lang_det_test_chrome
echo "--> cld2_unittest... "
./cld2_unittest > /dev/null
echo "--> cld2_unittest_avoid... "
./cld2_unittest_avoid > /dev/null
echo "--> compact_lang_det_test_chrome_2... "
echo "this is some english text" | ./compact_lang_det_test_chrome_2
echo "--> compact_lang_det_test_chrome_16... "
echo "this is some english text" | ./compact_lang_det_test_chrome_16
echo "--> cld2_unittest_chrome_2... "
./cld2_unittest_chrome_2 > /dev/null
echo "--> cld2_unittest_avoid_chrome_2... "
./cld2_unittest_avoid_chrome_2 > /dev/null
# ----------------------------------------------------------------------------
echo "--> [3 of 7] Invoking: compile_libs.sh..."
@@ -56,4 +58,4 @@ echo "--> cld2_dynamic_unittest... "
./cld2_dynamic_unittest --data-file cld2_data.bin > /dev/null
trap - INT TERM EXIT
echo "All libraries compiled and all tests passed!"
echo "All libraries compiled and all tests passed!"

4
internal/compile_dynamic.sh
View File

@@ -26,8 +26,8 @@ g++ -O2 -m64 cld2_dynamic_data_tool.cc \
tote.cc utf8statetable.cc \
cld_generated_cjk_uni_prop_80.cc cld2_generated_cjk_compatible.cc \
cld_generated_cjk_delta_bi_4.cc generated_distinct_bi_0.cc \
cld2_generated_quadchrome20141015_2.cc cld2_generated_deltaoctachrome20141015.cc \
cld2_generated_distinctoctachrome20141015.cc cld_generated_score_quad_octa_20141015_2.cc \
cld2_generated_quadchrome_2.cc cld2_generated_deltaoctachrome.cc \
cld2_generated_distinctoctachrome.cc cld_generated_score_quad_octa_2.cc \
-o cld2_dynamic_data_tool
echo " cld2_dynamic_data_tool compiled"

4
internal/compile_libs.sh
View File

@@ -22,8 +22,8 @@ g++ -shared -fPIC -O2 -m64 \
tote.cc utf8statetable.cc \
cld_generated_cjk_uni_prop_80.cc cld2_generated_cjk_compatible.cc \
cld_generated_cjk_delta_bi_4.cc generated_distinct_bi_0.cc \
cld2_generated_quadchrome0122_2.cc cld2_generated_deltaoctachrome0122.cc \
cld2_generated_distinctoctachrome0122.cc cld_generated_score_quad_octa_0122_2.cc \
cld2_generated_quadchrome_2.cc cld2_generated_deltaoctachrome.cc \
cld2_generated_distinctoctachrome.cc cld_generated_score_quad_octa_2.cc \
-o libcld2.so
g++ -shared -fPIC -O2 -m64 \