c9f9d40322
git-svn-id: https://cld2.googlecode.com/svn/trunk@168 b252ecd4-b096-bf77-eb8e-91563289f87e
1335 lines
50 KiB
C++
1335 lines
50 KiB
C++
// Copyright 2013 Google Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
//
|
|
// Author: dsites@google.com (Dick Sites)
|
|
// Updated 2014.01 for dual table lookup
|
|
//
|
|
|
|
#include "scoreonescriptspan.h"
|
|
|
|
#include "cldutil.h"
|
|
#include "debug.h"
|
|
#include "lang_script.h"
|
|
|
|
#include <stdio.h>
|
|
|
|
using namespace std;
|
|
|
|
namespace CLD2 {
|
|
|
|
static const int kUnreliablePercentThreshold = 75;
|
|
|
|
void AddLangProb(uint32 langprob, Tote* chunk_tote) {
|
|
ProcessProbV2Tote(langprob, chunk_tote);
|
|
}
|
|
|
|
void ZeroPSLang(uint32 langprob, Tote* chunk_tote) {
|
|
uint8 top1 = (langprob >> 8) & 0xff;
|
|
chunk_tote->SetScore(top1, 0);
|
|
}
|
|
|
|
bool SameCloseSet(uint16 lang1, uint16 lang2) {
|
|
int lang1_close_set = LanguageCloseSet(static_cast<Language>(lang1));
|
|
if (lang1_close_set == 0) {return false;}
|
|
int lang2_close_set = LanguageCloseSet(static_cast<Language>(lang2));
|
|
return (lang1_close_set == lang2_close_set);
|
|
}
|
|
|
|
bool SameCloseSet(Language lang1, Language lang2) {
|
|
int lang1_close_set = LanguageCloseSet(lang1);
|
|
if (lang1_close_set == 0) {return false;}
|
|
int lang2_close_set = LanguageCloseSet(lang2);
|
|
return (lang1_close_set == lang2_close_set);
|
|
}
|
|
|
|
|
|
// Needs expected score per 1KB in scoring context
|
|
void SetChunkSummary(ULScript ulscript, int first_linear_in_chunk,
|
|
int offset, int len,
|
|
const ScoringContext* scoringcontext,
|
|
const Tote* chunk_tote,
|
|
ChunkSummary* chunksummary) {
|
|
int key3[3];
|
|
chunk_tote->CurrentTopThreeKeys(key3);
|
|
Language lang1 = FromPerScriptNumber(ulscript, key3[0]);
|
|
Language lang2 = FromPerScriptNumber(ulscript, key3[1]);
|
|
|
|
int actual_score_per_kb = 0;
|
|
if (len > 0) {
|
|
actual_score_per_kb = (chunk_tote->GetScore(key3[0]) << 10) / len;
|
|
}
|
|
int expected_subscr = lang1 * 4 + LScript4(ulscript);
|
|
int expected_score_per_kb =
|
|
scoringcontext->scoringtables->kExpectedScore[expected_subscr];
|
|
|
|
chunksummary->offset = offset;
|
|
chunksummary->chunk_start = first_linear_in_chunk;
|
|
chunksummary->lang1 = lang1;
|
|
chunksummary->lang2 = lang2;
|
|
chunksummary->score1 = chunk_tote->GetScore(key3[0]);
|
|
chunksummary->score2 = chunk_tote->GetScore(key3[1]);
|
|
chunksummary->bytes = len;
|
|
chunksummary->grams = chunk_tote->GetScoreCount();
|
|
chunksummary->ulscript = ulscript;
|
|
chunksummary->reliability_delta = ReliabilityDelta(chunksummary->score1,
|
|
chunksummary->score2,
|
|
chunksummary->grams);
|
|
// If lang1/lang2 in same close set, set delta reliability to 100%
|
|
if (SameCloseSet(lang1, lang2)) {
|
|
chunksummary->reliability_delta = 100;
|
|
}
|
|
chunksummary->reliability_score =
|
|
ReliabilityExpected(actual_score_per_kb, expected_score_per_kb);
|
|
}
|
|
|
|
// Return true if just lang1 is there: lang2=0 and lang3=0
|
|
bool IsSingleLang(uint32 langprob) {
|
|
// Probably a bug -- which end is lang1? But only used to call empty Boost1
|
|
return ((langprob & 0x00ffff00) == 0);
|
|
}
|
|
|
|
// Update scoring context distinct_boost for single language quad
|
|
void AddDistinctBoost1(uint32 langprob, ScoringContext* scoringcontext) {
|
|
// Probably keep this empty -- not a good enough signal
|
|
}
|
|
|
|
// Update scoring context distinct_boost for distinct octagram
|
|
// Keep last 4 used. Since these are mostly (except at splices) in
|
|
// hitbuffer, we might be able to just use a subscript and splice
|
|
void AddDistinctBoost2(uint32 langprob, ScoringContext* scoringcontext) {
|
|
// this is called 0..n times per chunk with decoded hitbuffer->distinct...
|
|
LangBoosts* distinct_boost = &scoringcontext->distinct_boost.latn;
|
|
if (scoringcontext->ulscript != ULScript_Latin) {
|
|
distinct_boost = &scoringcontext->distinct_boost.othr;
|
|
}
|
|
int n = distinct_boost->n;
|
|
distinct_boost->langprob[n] = langprob;
|
|
distinct_boost->n = distinct_boost->wrap(n + 1);
|
|
}
|
|
|
|
// For each chunk, add extra weight for language priors (from content-lang and
|
|
// meta lang=xx) and distinctive tokens
|
|
void ScoreBoosts(const ScoringContext* scoringcontext, Tote* chunk_tote) {
|
|
// Get boosts for current script
|
|
const LangBoosts* langprior_boost = &scoringcontext->langprior_boost.latn;
|
|
const LangBoosts* langprior_whack = &scoringcontext->langprior_whack.latn;
|
|
const LangBoosts* distinct_boost = &scoringcontext->distinct_boost.latn;
|
|
if (scoringcontext->ulscript != ULScript_Latin) {
|
|
langprior_boost = &scoringcontext->langprior_boost.othr;
|
|
langprior_whack = &scoringcontext->langprior_whack.othr;
|
|
distinct_boost = &scoringcontext->distinct_boost.othr;
|
|
}
|
|
|
|
for (int k = 0; k < kMaxBoosts; ++k) {
|
|
uint32 langprob = langprior_boost->langprob[k];
|
|
if (langprob > 0) {AddLangProb(langprob, chunk_tote);}
|
|
}
|
|
for (int k = 0; k < kMaxBoosts; ++k) {
|
|
uint32 langprob = distinct_boost->langprob[k];
|
|
if (langprob > 0) {AddLangProb(langprob, chunk_tote);}
|
|
}
|
|
// boost has a packed set of per-script langs and probabilites
|
|
// whack has a packed set of per-script lang to be suppressed (zeroed)
|
|
// When a language in a close set is given as an explicit hint, others in
|
|
// that set will be whacked here.
|
|
for (int k = 0; k < kMaxBoosts; ++k) {
|
|
uint32 langprob = langprior_whack->langprob[k];
|
|
if (langprob > 0) {ZeroPSLang(langprob, chunk_tote);}
|
|
}
|
|
}
|
|
|
|
|
|
|
|
// At this point, The chunk is described by
|
|
// hitbuffer->base[cspan->chunk_base .. cspan->chunk_base + cspan->base_len)
|
|
// hitbuffer->delta[cspan->chunk_delta ... )
|
|
// hitbuffer->distinct[cspan->chunk_distinct ... )
|
|
// Scored text is in text[lo..hi) where
|
|
// lo is 0 or the min of first base/delta/distinct hitbuffer offset and
|
|
// hi is the min of next base/delta/distinct hitbuffer offset after
|
|
// base_len, etc.
|
|
void GetTextSpanOffsets(const ScoringHitBuffer* hitbuffer,
|
|
const ChunkSpan* cspan, int* lo, int* hi) {
|
|
// Front of this span
|
|
int lo_base = hitbuffer->base[cspan->chunk_base].offset;
|
|
int lo_delta = hitbuffer->delta[cspan->chunk_delta].offset;
|
|
int lo_distinct = hitbuffer->distinct[cspan->chunk_distinct].offset;
|
|
// Front of next span
|
|
int hi_base = hitbuffer->base[cspan->chunk_base +
|
|
cspan->base_len].offset;
|
|
int hi_delta = hitbuffer->delta[cspan->chunk_delta +
|
|
cspan->delta_len].offset;
|
|
int hi_distinct = hitbuffer->distinct[cspan->chunk_distinct +
|
|
cspan->distinct_len].offset;
|
|
|
|
*lo = 0;
|
|
// if (cspan->chunk_base > 0) {
|
|
// *lo = minint(minint(lo_base, lo_delta), lo_distinct);
|
|
// }
|
|
*lo = minint(minint(lo_base, lo_delta), lo_distinct);
|
|
*hi = minint(minint(hi_base, hi_delta), hi_distinct);
|
|
}
|
|
|
|
|
|
int DiffScore(const CLD2TableSummary* obj, int indirect,
|
|
uint8 lang1, uint8 lang2) {
|
|
if (indirect < static_cast<int>(obj->kCLDTableSizeOne)) {
|
|
// Up to three languages at indirect
|
|
uint32 langprob = obj->kCLDTableInd[indirect];
|
|
return GetLangScore(langprob, lang1) - GetLangScore(langprob, lang2);
|
|
} else {
|
|
// Up to six languages at start + 2 * (indirect - start)
|
|
indirect += (indirect - obj->kCLDTableSizeOne);
|
|
uint32 langprob = obj->kCLDTableInd[indirect];
|
|
uint32 langprob2 = obj->kCLDTableInd[indirect + 1];
|
|
return (GetLangScore(langprob, lang1) + GetLangScore(langprob2, lang1)) -
|
|
(GetLangScore(langprob, lang2) + GetLangScore(langprob2, lang2));
|
|
}
|
|
|
|
}
|
|
|
|
// Score all the bases, deltas, distincts, boosts for one chunk into chunk_tote
|
|
// After last chunk there is always a hitbuffer entry with an offset just off
|
|
// the end of the text.
|
|
// Sets delta_len, and distinct_len
|
|
void ScoreOneChunk(const char* text, ULScript ulscript,
|
|
const ScoringHitBuffer* hitbuffer,
|
|
int chunk_i,
|
|
ScoringContext* scoringcontext,
|
|
ChunkSpan* cspan, Tote* chunk_tote,
|
|
ChunkSummary* chunksummary) {
|
|
int first_linear_in_chunk = hitbuffer->chunk_start[chunk_i];
|
|
int first_linear_in_next_chunk = hitbuffer->chunk_start[chunk_i + 1];
|
|
|
|
chunk_tote->Reinit();
|
|
cspan->delta_len = 0;
|
|
cspan->distinct_len = 0;
|
|
if (scoringcontext->flags_cld2_verbose) {
|
|
fprintf(scoringcontext->debug_file, "<br>ScoreOneChunk[%d..%d) ",
|
|
first_linear_in_chunk, first_linear_in_next_chunk);
|
|
}
|
|
|
|
// 2013.02.05 linear design: just use base and base_len for the span
|
|
cspan->chunk_base = first_linear_in_chunk;
|
|
cspan->base_len = first_linear_in_next_chunk - first_linear_in_chunk;
|
|
for (int i = first_linear_in_chunk; i < first_linear_in_next_chunk; ++i) {
|
|
uint32 langprob = hitbuffer->linear[i].langprob;
|
|
AddLangProb(langprob, chunk_tote);
|
|
if (hitbuffer->linear[i].type <= QUADHIT) {
|
|
chunk_tote->AddScoreCount(); // Just count quads, not octas
|
|
}
|
|
if (hitbuffer->linear[i].type == DISTINCTHIT) {
|
|
AddDistinctBoost2(langprob, scoringcontext);
|
|
}
|
|
}
|
|
|
|
// Score language prior boosts
|
|
// Score distinct word boost
|
|
ScoreBoosts(scoringcontext, chunk_tote);
|
|
|
|
int lo = hitbuffer->linear[first_linear_in_chunk].offset;
|
|
int hi = hitbuffer->linear[first_linear_in_next_chunk].offset;
|
|
|
|
// Chunk_tote: get top langs, scores, etc. and fill in chunk summary
|
|
SetChunkSummary(ulscript, first_linear_in_chunk, lo, hi - lo,
|
|
scoringcontext, chunk_tote, chunksummary);
|
|
|
|
bool more_to_come = false;
|
|
bool score_cjk = false;
|
|
if (scoringcontext->flags_cld2_html) {
|
|
// Show one chunk in readable output
|
|
CLD2_Debug(text, lo, hi, more_to_come, score_cjk, hitbuffer,
|
|
scoringcontext, cspan, chunksummary);
|
|
}
|
|
|
|
scoringcontext->prior_chunk_lang = static_cast<Language>(chunksummary->lang1);
|
|
}
|
|
|
|
|
|
// Score chunks of text described by hitbuffer, allowing each to be in a
|
|
// different language, and optionally adjusting the boundaries inbetween.
|
|
// Set last_cspan to the last chunkspan used
|
|
void ScoreAllHits(const char* text, ULScript ulscript,
|
|
bool more_to_come, bool score_cjk,
|
|
const ScoringHitBuffer* hitbuffer,
|
|
ScoringContext* scoringcontext,
|
|
SummaryBuffer* summarybuffer, ChunkSpan* last_cspan) {
|
|
ChunkSpan prior_cspan = {0, 0, 0, 0, 0, 0};
|
|
ChunkSpan cspan = {0, 0, 0, 0, 0, 0};
|
|
|
|
for (int i = 0; i < hitbuffer->next_chunk_start; ++i) {
|
|
// Score one chunk
|
|
// Sets delta_len, and distinct_len
|
|
Tote chunk_tote;
|
|
ChunkSummary chunksummary;
|
|
ScoreOneChunk(text, ulscript,
|
|
hitbuffer, i,
|
|
scoringcontext, &cspan, &chunk_tote, &chunksummary);
|
|
|
|
// Put result in summarybuffer
|
|
if (summarybuffer->n < kMaxSummaries) {
|
|
summarybuffer->chunksummary[summarybuffer->n] = chunksummary;
|
|
summarybuffer->n += 1;
|
|
}
|
|
|
|
prior_cspan = cspan;
|
|
cspan.chunk_base += cspan.base_len;
|
|
cspan.chunk_delta += cspan.delta_len;
|
|
cspan.chunk_distinct += cspan.distinct_len;
|
|
}
|
|
|
|
// Add one dummy off the end to hold first unused linear_in_chunk
|
|
int linear_off_end = hitbuffer->next_linear;
|
|
int offset_off_end = hitbuffer->linear[linear_off_end].offset;
|
|
ChunkSummary* cs = &summarybuffer->chunksummary[summarybuffer->n];
|
|
memset(cs, 0, sizeof(ChunkSummary));
|
|
cs->offset = offset_off_end;
|
|
cs->chunk_start = linear_off_end;
|
|
*last_cspan = prior_cspan;
|
|
}
|
|
|
|
|
|
void SummaryBufferToDocTote(const SummaryBuffer* summarybuffer,
|
|
bool more_to_come, DocTote* doc_tote) {
|
|
int cs_bytes_sum = 0;
|
|
for (int i = 0; i < summarybuffer->n; ++i) {
|
|
const ChunkSummary* cs = &summarybuffer->chunksummary[i];
|
|
int reliability = minint(cs->reliability_delta, cs->reliability_score);
|
|
// doc_tote uses full languages
|
|
doc_tote->Add(cs->lang1, cs->bytes, cs->score1, reliability);
|
|
cs_bytes_sum += cs->bytes;
|
|
}
|
|
}
|
|
|
|
// Turn on for debugging vectors
|
|
static const bool kShowLettersOriginal = false;
|
|
|
|
|
|
// If next chunk language matches last vector language, extend last element
|
|
// Otherwise add new element to vector
|
|
void ItemToVector(ScriptScanner* scanner,
|
|
ResultChunkVector* vec, Language new_lang,
|
|
int mapped_offset, int mapped_len) {
|
|
uint16 last_vec_lang = static_cast<uint16>(UNKNOWN_LANGUAGE);
|
|
int last_vec_subscr = vec->size() - 1;
|
|
if (last_vec_subscr >= 0) {
|
|
ResultChunk* priorrc = &(*vec)[last_vec_subscr];
|
|
last_vec_lang = priorrc->lang1;
|
|
if (new_lang == last_vec_lang) {
|
|
// Extend prior. Current mapped_offset may be beyond prior end, so do
|
|
// the arithmetic to include any such gap
|
|
priorrc->bytes = minint((mapped_offset + mapped_len) - priorrc->offset,
|
|
kMaxResultChunkBytes);
|
|
if (kShowLettersOriginal) {
|
|
// Optionally print the new chunk original text
|
|
string temp2(&scanner->GetBufferStart()[priorrc->offset],
|
|
priorrc->bytes);
|
|
fprintf(stderr, "Item[%d..%d) '%s'<br>\n",
|
|
priorrc->offset, priorrc->offset + priorrc->bytes,
|
|
GetHtmlEscapedText(temp2).c_str());
|
|
}
|
|
return;
|
|
}
|
|
}
|
|
// Add new vector element
|
|
ResultChunk rc;
|
|
rc.offset = mapped_offset;
|
|
rc.bytes = minint(mapped_len, kMaxResultChunkBytes);
|
|
rc.lang1 = static_cast<uint16>(new_lang);
|
|
vec->push_back(rc);
|
|
if (kShowLettersOriginal) {
|
|
// Optionally print the new chunk original text
|
|
string temp2(&scanner->GetBufferStart()[rc.offset], rc.bytes);
|
|
fprintf(stderr, "Item[%d..%d) '%s'<br>\n",
|
|
rc.offset, rc.offset + rc.bytes,
|
|
GetHtmlEscapedText(temp2).c_str());
|
|
}
|
|
}
|
|
|
|
uint16 PriorVecLang(const ResultChunkVector* vec) {
|
|
if (vec->empty()) {return static_cast<uint16>(UNKNOWN_LANGUAGE);}
|
|
return (*vec)[vec->size() - 1].lang1;
|
|
}
|
|
|
|
uint16 NextChunkLang(const SummaryBuffer* summarybuffer, int i) {
|
|
if ((i + 1) >= summarybuffer->n) {
|
|
return static_cast<uint16>(UNKNOWN_LANGUAGE);
|
|
}
|
|
return summarybuffer->chunksummary[i + 1].lang1;
|
|
}
|
|
|
|
|
|
|
|
// Add n elements of summarybuffer to resultchunk vector:
|
|
// Each element is letters-only text [offset..offset+bytes)
|
|
// This maps back to original[Back(offset)..Back(offset+bytes))
|
|
//
|
|
// We go out of our way to minimize the variation in the ResultChunkVector,
|
|
// so that the caller has fewer but more meaningful spans in different
|
|
// lanaguges, for the likely purpose of translation or spell-check.
|
|
//
|
|
// The language of each chunk is lang1, but it might be unreliable for
|
|
// either of two reasons: its score is relatively too close to the score of
|
|
// lang2, or its score is too far away from the expected score of real text in
|
|
// the given language. Unreliable languages are mapped to Unknown.
|
|
//
|
|
void SummaryBufferToVector(ScriptScanner* scanner, const char* text,
|
|
const SummaryBuffer* summarybuffer,
|
|
bool more_to_come, ResultChunkVector* vec) {
|
|
if (vec == NULL) {return;}
|
|
|
|
if (kShowLettersOriginal) {
|
|
fprintf(stderr, "map2original_ ");
|
|
scanner->map2original_.DumpWindow();
|
|
fprintf(stderr, "<br>\n");
|
|
fprintf(stderr, "map2uplow_ ");
|
|
scanner->map2uplow_.DumpWindow();
|
|
fprintf(stderr, "<br>\n");
|
|
}
|
|
|
|
for (int i = 0; i < summarybuffer->n; ++i) {
|
|
const ChunkSummary* cs = &summarybuffer->chunksummary[i];
|
|
int unmapped_offset = cs->offset;
|
|
int unmapped_len = cs->bytes;
|
|
|
|
if (kShowLettersOriginal) {
|
|
// Optionally print the chunk lowercase letters/marks text
|
|
string temp(&text[unmapped_offset], unmapped_len);
|
|
fprintf(stderr, "Letters [%d..%d) '%s'<br>\n",
|
|
unmapped_offset, unmapped_offset + unmapped_len,
|
|
GetHtmlEscapedText(temp).c_str());
|
|
}
|
|
|
|
int mapped_offset = scanner->MapBack(unmapped_offset);
|
|
|
|
// Trim back a little to prefer splicing original at word boundaries
|
|
if (mapped_offset > 0) {
|
|
// Size of prior vector entry, if any
|
|
int prior_size = 0;
|
|
if (!vec->empty()) {
|
|
ResultChunk* rc = &(*vec)[vec->size() - 1];
|
|
prior_size = rc->bytes;
|
|
}
|
|
// Maximum back up size to leave at least 3 bytes in prior,
|
|
// and not entire buffer, and no more than 12 bytes total backup
|
|
int n_limit = minint(prior_size - 3, mapped_offset);
|
|
n_limit = minint(n_limit, 12);
|
|
|
|
// Backscan over letters, stopping if prior byte is < 0x41
|
|
// There is some possibility that we will backscan over a different script
|
|
const char* s = &scanner->GetBufferStart()[mapped_offset];
|
|
const unsigned char* us = reinterpret_cast<const unsigned char*>(s);
|
|
int n = 0;
|
|
while ((n < n_limit) && (us[-n - 1] >= 0x41)) {++n;}
|
|
if (n >= n_limit) {n = 0;} // New boundary not found within range
|
|
|
|
// Also back up exactly one leading punctuation character if '"#@
|
|
if (n < n_limit) {
|
|
unsigned char c = us[-n - 1];
|
|
if ((c == '\'') || (c == '"') || (c == '#') || (c == '@')) {++n;}
|
|
}
|
|
// Shrink the previous chunk slightly
|
|
if (n > 0) {
|
|
ResultChunk* rc = &(*vec)[vec->size() - 1];
|
|
rc->bytes -= n;
|
|
mapped_offset -= n;
|
|
if (kShowLettersOriginal) {
|
|
fprintf(stderr, "Back up %d bytes<br>\n", n);
|
|
// Optionally print the prior chunk original text
|
|
string temp2(&scanner->GetBufferStart()[rc->offset], rc->bytes);
|
|
fprintf(stderr, "Prior [%d..%d) '%s'<br>\n",
|
|
rc->offset, rc->offset + rc->bytes,
|
|
GetHtmlEscapedText(temp2).c_str());
|
|
}
|
|
}
|
|
}
|
|
|
|
int mapped_len =
|
|
scanner->MapBack(unmapped_offset + unmapped_len) - mapped_offset;
|
|
|
|
if (kShowLettersOriginal) {
|
|
// Optionally print the chunk original text
|
|
string temp2(&scanner->GetBufferStart()[mapped_offset], mapped_len);
|
|
fprintf(stderr, "Original[%d..%d) '%s'<br>\n",
|
|
mapped_offset, mapped_offset + mapped_len,
|
|
GetHtmlEscapedText(temp2).c_str());
|
|
}
|
|
|
|
Language new_lang = static_cast<Language>(cs->lang1);
|
|
bool reliability_delta_bad =
|
|
(cs->reliability_delta < kUnreliablePercentThreshold);
|
|
bool reliability_score_bad =
|
|
(cs->reliability_score < kUnreliablePercentThreshold);
|
|
|
|
// If the top language matches last vector, ignore reliability_delta
|
|
uint16 prior_lang = PriorVecLang(vec);
|
|
if (prior_lang == cs->lang1) {
|
|
reliability_delta_bad = false;
|
|
}
|
|
// If the top language is in same close set as last vector, set up to merge
|
|
if (SameCloseSet(cs->lang1, prior_lang)) {
|
|
new_lang = static_cast<Language>(prior_lang);
|
|
reliability_delta_bad = false;
|
|
}
|
|
// If the top two languages are in the same close set and the last vector
|
|
// language is the second language, set up to merge
|
|
if (SameCloseSet(cs->lang1, cs->lang2) &&
|
|
(prior_lang == cs->lang2)) {
|
|
new_lang = static_cast<Language>(prior_lang);
|
|
reliability_delta_bad = false;
|
|
}
|
|
// If unreliable and the last and next vector languages are both
|
|
// the second language, set up to merge
|
|
uint16 next_lang = NextChunkLang(summarybuffer, i);
|
|
if (reliability_delta_bad &&
|
|
(prior_lang == cs->lang2) && (next_lang == cs->lang2)) {
|
|
new_lang = static_cast<Language>(prior_lang);
|
|
reliability_delta_bad = false;
|
|
}
|
|
|
|
if (reliability_delta_bad || reliability_score_bad) {
|
|
new_lang = UNKNOWN_LANGUAGE;
|
|
}
|
|
ItemToVector(scanner, vec, new_lang, mapped_offset, mapped_len);
|
|
}
|
|
}
|
|
|
|
// Add just one element to resultchunk vector:
|
|
// For RTypeNone or RTypeOne
|
|
void JustOneItemToVector(ScriptScanner* scanner, const char* text,
|
|
Language lang1, int unmapped_offset, int unmapped_len,
|
|
ResultChunkVector* vec) {
|
|
if (vec == NULL) {return;}
|
|
|
|
if (kShowLettersOriginal) {
|
|
fprintf(stderr, "map2original_ ");
|
|
scanner->map2original_.DumpWindow();
|
|
fprintf(stderr, "<br>\n");
|
|
fprintf(stderr, "map2uplow_ ");
|
|
scanner->map2uplow_.DumpWindow();
|
|
fprintf(stderr, "<br>\n");
|
|
}
|
|
|
|
if (kShowLettersOriginal) {
|
|
// Optionally print the chunk lowercase letters/marks text
|
|
string temp(&text[unmapped_offset], unmapped_len);
|
|
fprintf(stderr, "Letters1 [%d..%d) '%s'<br>\n",
|
|
unmapped_offset, unmapped_offset + unmapped_len,
|
|
GetHtmlEscapedText(temp).c_str());
|
|
}
|
|
|
|
int mapped_offset = scanner->MapBack(unmapped_offset);
|
|
int mapped_len =
|
|
scanner->MapBack(unmapped_offset + unmapped_len) - mapped_offset;
|
|
|
|
if (kShowLettersOriginal) {
|
|
// Optionally print the chunk original text
|
|
string temp2(&scanner->GetBufferStart()[mapped_offset], mapped_len);
|
|
fprintf(stderr, "Original1[%d..%d) '%s'<br>\n",
|
|
mapped_offset, mapped_offset + mapped_len,
|
|
GetHtmlEscapedText(temp2).c_str());
|
|
}
|
|
|
|
ItemToVector(scanner, vec, lang1, mapped_offset, mapped_len);
|
|
}
|
|
|
|
|
|
// Debugging. Not thread safe. Defined in getonescriptspan
|
|
char* DisplayPiece(const char* next_byte_, int byte_length_);
|
|
|
|
// If high bit is on, take out high bit and add 2B to make table2 entries easy
|
|
inline int PrintableIndirect(int x) {
|
|
if ((x & 0x80000000u) != 0) {
|
|
return (x & ~0x80000000u) + 2000000000;
|
|
}
|
|
return x;
|
|
}
|
|
void DumpHitBuffer(FILE* df, const char* text,
|
|
const ScoringHitBuffer* hitbuffer) {
|
|
fprintf(df,
|
|
"<br>DumpHitBuffer[%s, next_base/delta/distinct %d, %d, %d)<br>\n",
|
|
ULScriptCode(hitbuffer->ulscript),
|
|
hitbuffer->next_base, hitbuffer->next_delta,
|
|
hitbuffer->next_distinct);
|
|
for (int i = 0; i < hitbuffer->maxscoringhits; ++i) {
|
|
if (i < hitbuffer->next_base) {
|
|
fprintf(df, "Q[%d]%d,%d,%s ",
|
|
i, hitbuffer->base[i].offset,
|
|
PrintableIndirect(hitbuffer->base[i].indirect),
|
|
DisplayPiece(&text[hitbuffer->base[i].offset], 6));
|
|
}
|
|
if (i < hitbuffer->next_delta) {
|
|
fprintf(df, "DL[%d]%d,%d,%s ",
|
|
i, hitbuffer->delta[i].offset, hitbuffer->delta[i].indirect,
|
|
DisplayPiece(&text[hitbuffer->delta[i].offset], 12));
|
|
}
|
|
if (i < hitbuffer->next_distinct) {
|
|
fprintf(df, "D[%d]%d,%d,%s ",
|
|
i, hitbuffer->distinct[i].offset, hitbuffer->distinct[i].indirect,
|
|
DisplayPiece(&text[hitbuffer->distinct[i].offset], 12));
|
|
}
|
|
if (i < hitbuffer->next_base) {
|
|
fprintf(df, "<br>\n");
|
|
}
|
|
if (i > 50) {break;}
|
|
}
|
|
if (hitbuffer->next_base > 50) {
|
|
int i = hitbuffer->next_base;
|
|
fprintf(df, "Q[%d]%d,%d,%s ",
|
|
i, hitbuffer->base[i].offset,
|
|
PrintableIndirect(hitbuffer->base[i].indirect),
|
|
DisplayPiece(&text[hitbuffer->base[i].offset], 6));
|
|
}
|
|
if (hitbuffer->next_delta > 50) {
|
|
int i = hitbuffer->next_delta;
|
|
fprintf(df, "DL[%d]%d,%d,%s ",
|
|
i, hitbuffer->delta[i].offset, hitbuffer->delta[i].indirect,
|
|
DisplayPiece(&text[hitbuffer->delta[i].offset], 12));
|
|
}
|
|
if (hitbuffer->next_distinct > 50) {
|
|
int i = hitbuffer->next_distinct;
|
|
fprintf(df, "D[%d]%d,%d,%s ",
|
|
i, hitbuffer->distinct[i].offset, hitbuffer->distinct[i].indirect,
|
|
DisplayPiece(&text[hitbuffer->distinct[i].offset], 12));
|
|
}
|
|
fprintf(df, "<br>\n");
|
|
}
|
|
|
|
|
|
void DumpLinearBuffer(FILE* df, const char* text,
|
|
const ScoringHitBuffer* hitbuffer) {
|
|
fprintf(df, "<br>DumpLinearBuffer[%d)<br>\n",
|
|
hitbuffer->next_linear);
|
|
// Include the dummy entry off the end
|
|
for (int i = 0; i < hitbuffer->next_linear + 1; ++i) {
|
|
if ((50 < i) && (i < (hitbuffer->next_linear - 1))) {continue;}
|
|
fprintf(df, "[%d]%d,%c=%08x,%s<br>\n",
|
|
i, hitbuffer->linear[i].offset,
|
|
"UQLD"[hitbuffer->linear[i].type],
|
|
hitbuffer->linear[i].langprob,
|
|
DisplayPiece(&text[hitbuffer->linear[i].offset], 6));
|
|
}
|
|
fprintf(df, "<br>\n");
|
|
|
|
fprintf(df, "DumpChunkStart[%d]<br>\n", hitbuffer->next_chunk_start);
|
|
for (int i = 0; i < hitbuffer->next_chunk_start + 1; ++i) {
|
|
fprintf(df, "[%d]%d\n", i, hitbuffer->chunk_start[i]);
|
|
}
|
|
fprintf(df, "<br>\n");
|
|
}
|
|
|
|
// Move this verbose debugging output to debug.cc eventually
|
|
void DumpChunkSummary(FILE* df, const ChunkSummary* cs) {
|
|
// Print chunksummary
|
|
fprintf(df, "%d lin[%d] %s.%d %s.%d %dB %d# %s %dRd %dRs<br>\n",
|
|
cs->offset,
|
|
cs->chunk_start,
|
|
LanguageCode(static_cast<Language>(cs->lang1)),
|
|
cs->score1,
|
|
LanguageCode(static_cast<Language>(cs->lang2)),
|
|
cs->score2,
|
|
cs->bytes,
|
|
cs->grams,
|
|
ULScriptCode(static_cast<ULScript>(cs->ulscript)),
|
|
cs->reliability_delta,
|
|
cs->reliability_score);
|
|
}
|
|
|
|
void DumpSummaryBuffer(FILE* df, const SummaryBuffer* summarybuffer) {
|
|
fprintf(df, "<br>DumpSummaryBuffer[%d]<br>\n", summarybuffer->n);
|
|
fprintf(df, "[i] offset linear[chunk_start] lang.score1 lang.score2 "
|
|
"bytesB ngrams# script rel_delta rel_score<br>\n");
|
|
for (int i = 0; i <= summarybuffer->n; ++i) {
|
|
fprintf(df, "[%d] ", i);
|
|
DumpChunkSummary(df, &summarybuffer->chunksummary[i]);
|
|
}
|
|
fprintf(df, "<br>\n");
|
|
}
|
|
|
|
|
|
|
|
// Within hitbufer->linear[]
|
|
// <-- prior chunk --><-- this chunk -->
|
|
// | | |
|
|
// linear0 linear1 linear2
|
|
// lang0 lang1
|
|
// The goal of sharpening is to move this_linear to better separate langs
|
|
int BetterBoundary(const char* text,
|
|
ScoringHitBuffer* hitbuffer,
|
|
ScoringContext* scoringcontext,
|
|
uint8 pslang0, uint8 pslang1,
|
|
int linear0, int linear1, int linear2) {
|
|
// Degenerate case, no change
|
|
if ((linear2 - linear0) <= 8) {return linear1;}
|
|
|
|
// Each diff gives pslang0 score - pslang1 score
|
|
// Running diff has four entries + + + + followed by four entries - - - -
|
|
// so that this value is maximal at the sharpest boundary between pslang0
|
|
// (positive diffs) and pslang1 (negative diffs)
|
|
int running_diff = 0;
|
|
int diff[8]; // Ring buffer of pslang0-pslang1 differences
|
|
// Initialize with first 8 diffs
|
|
for (int i = linear0; i < linear0 + 8; ++i) {
|
|
int j = i & 7;
|
|
uint32 langprob = hitbuffer->linear[i].langprob;
|
|
diff[j] = GetLangScore(langprob, pslang0) -
|
|
GetLangScore(langprob, pslang1);
|
|
if (i < linear0 + 4) {
|
|
// First four diffs pslang0 - pslang1
|
|
running_diff += diff[j];
|
|
} else {
|
|
// Second four diffs -(pslang0 - pslang1)
|
|
running_diff -= diff[j];
|
|
}
|
|
}
|
|
|
|
// Now scan for sharpest boundary. j is at left end of 8 entries
|
|
// To be a boundary, there must be both >0 and <0 entries in the window
|
|
int better_boundary_value = 0;
|
|
int better_boundary = linear1;
|
|
for (int i = linear0; i < linear2 - 8; ++i) {
|
|
int j = i & 7;
|
|
if (better_boundary_value < running_diff) {
|
|
bool has_plus = false;
|
|
bool has_minus = false;
|
|
for (int kk = 0; kk < 8; ++kk) {
|
|
if (diff[kk] > 0) {has_plus = true;}
|
|
if (diff[kk] < 0) {has_minus = true;}
|
|
}
|
|
if (has_plus && has_minus) {
|
|
better_boundary_value = running_diff;
|
|
better_boundary = i + 4;
|
|
}
|
|
}
|
|
// Shift right one entry
|
|
uint32 langprob = hitbuffer->linear[i + 8].langprob;
|
|
int newdiff = GetLangScore(langprob, pslang0) -
|
|
GetLangScore(langprob, pslang1);
|
|
int middiff = diff[(i + 4) & 7];
|
|
int olddiff = diff[j];
|
|
diff[j] = newdiff;
|
|
running_diff -= olddiff; // Remove left
|
|
running_diff += 2 * middiff; // Convert middle from - to +
|
|
running_diff -= newdiff; // Insert right
|
|
}
|
|
|
|
if (scoringcontext->flags_cld2_verbose && (linear1 != better_boundary)) {
|
|
Language lang0 = FromPerScriptNumber(scoringcontext->ulscript, pslang0);
|
|
Language lang1 = FromPerScriptNumber(scoringcontext->ulscript, pslang1);
|
|
fprintf(scoringcontext->debug_file, " Better lin[%d=>%d] %s^^%s <br>\n",
|
|
linear1, better_boundary,
|
|
LanguageCode(lang0), LanguageCode(lang1));
|
|
int lin0_off = hitbuffer->linear[linear0].offset;
|
|
int lin1_off = hitbuffer->linear[linear1].offset;
|
|
int lin2_off = hitbuffer->linear[linear2].offset;
|
|
int better_offm1 = hitbuffer->linear[better_boundary - 1].offset;
|
|
int better_off = hitbuffer->linear[better_boundary].offset;
|
|
int better_offp1 = hitbuffer->linear[better_boundary + 1].offset;
|
|
string old0(&text[lin0_off], lin1_off - lin0_off);
|
|
string old1(&text[lin1_off], lin2_off - lin1_off);
|
|
string new0(&text[lin0_off], better_offm1 - lin0_off);
|
|
string new0m1(&text[better_offm1], better_off - better_offm1);
|
|
string new1(&text[better_off], better_offp1 - better_off);
|
|
string new1p1(&text[better_offp1], lin2_off - better_offp1);
|
|
fprintf(scoringcontext->debug_file, "%s^^%s => <br>\n%s^%s^^%s^%s<br>\n",
|
|
GetHtmlEscapedText(old0).c_str(),
|
|
GetHtmlEscapedText(old1).c_str(),
|
|
GetHtmlEscapedText(new0).c_str(),
|
|
GetHtmlEscapedText(new0m1).c_str(),
|
|
GetHtmlEscapedText(new1).c_str(),
|
|
GetHtmlEscapedText(new1p1).c_str());
|
|
// Slow picture of differences per linear entry
|
|
int d;
|
|
for (int i = linear0; i < linear2; ++i) {
|
|
if (i == better_boundary) {
|
|
fprintf(scoringcontext->debug_file, "^^ ");
|
|
}
|
|
uint32 langprob = hitbuffer->linear[i].langprob;
|
|
d = GetLangScore(langprob, pslang0) - GetLangScore(langprob, pslang1);
|
|
const char* s = "=";
|
|
//if (d > 2) {s = "\xc2\xaf";} // Macron
|
|
if (d > 2) {s = "#";}
|
|
else if (d > 0) {s = "+";}
|
|
else if (d < -2) {s = "_";}
|
|
else if (d < 0) {s = "-";}
|
|
fprintf(scoringcontext->debug_file, "%s ", s);
|
|
}
|
|
fprintf(scoringcontext->debug_file, " (scale: #+=-_)<br>\n");
|
|
}
|
|
return better_boundary;
|
|
}
|
|
|
|
|
|
// For all but the first summary, if its top language differs from
|
|
// the previous chunk, refine the boundary
|
|
// Linearized version
|
|
void SharpenBoundaries(const char* text,
|
|
bool more_to_come,
|
|
ScoringHitBuffer* hitbuffer,
|
|
ScoringContext* scoringcontext,
|
|
SummaryBuffer* summarybuffer) {
|
|
|
|
int prior_linear = summarybuffer->chunksummary[0].chunk_start;
|
|
uint16 prior_lang = summarybuffer->chunksummary[0].lang1;
|
|
|
|
if (scoringcontext->flags_cld2_verbose) {
|
|
fprintf(scoringcontext->debug_file, "<br>SharpenBoundaries<br>\n");
|
|
}
|
|
for (int i = 1; i < summarybuffer->n; ++i) {
|
|
ChunkSummary* cs = &summarybuffer->chunksummary[i];
|
|
uint16 this_lang = cs->lang1;
|
|
if (this_lang == prior_lang) {
|
|
prior_linear = cs->chunk_start;
|
|
continue;
|
|
}
|
|
|
|
int this_linear = cs->chunk_start;
|
|
int next_linear = summarybuffer->chunksummary[i + 1].chunk_start;
|
|
|
|
// If this/prior in same close set, don't move boundary
|
|
if (SameCloseSet(prior_lang, this_lang)) {
|
|
prior_linear = this_linear;
|
|
prior_lang = this_lang;
|
|
continue;
|
|
}
|
|
|
|
|
|
// Within hitbuffer->linear[]
|
|
// <-- prior chunk --><-- this chunk -->
|
|
// | | |
|
|
// prior_linear this_linear next_linear
|
|
// prior_lang this_lang
|
|
// The goal of sharpening is to move this_linear to better separate langs
|
|
|
|
uint8 pslang0 = PerScriptNumber(scoringcontext->ulscript,
|
|
static_cast<Language>(prior_lang));
|
|
uint8 pslang1 = PerScriptNumber(scoringcontext->ulscript,
|
|
static_cast<Language>(this_lang));
|
|
int better_linear = BetterBoundary(text,
|
|
hitbuffer,
|
|
scoringcontext,
|
|
pslang0, pslang1,
|
|
prior_linear, this_linear, next_linear);
|
|
|
|
int old_offset = hitbuffer->linear[this_linear].offset;
|
|
int new_offset = hitbuffer->linear[better_linear].offset;
|
|
cs->chunk_start = better_linear;
|
|
cs->offset = new_offset;
|
|
// If this_linear moved right, make bytes smaller for this, larger for prior
|
|
// If this_linear moved left, make bytes larger for this, smaller for prior
|
|
cs->bytes -= (new_offset - old_offset);
|
|
summarybuffer->chunksummary[i - 1].bytes += (new_offset - old_offset);
|
|
|
|
this_linear = better_linear; // Update so that next chunk doesn't intrude
|
|
|
|
// Consider rescoring the two chunks
|
|
|
|
// Update for next round (note: using pre-updated boundary)
|
|
prior_linear = this_linear;
|
|
prior_lang = this_lang;
|
|
}
|
|
}
|
|
|
|
// Make a langprob that gives small weight to the default language for ulscript
|
|
uint32 DefaultLangProb(ULScript ulscript) {
|
|
Language default_lang = DefaultLanguage(ulscript);
|
|
return MakeLangProb(default_lang, 1);
|
|
}
|
|
|
|
// Effectively, do a merge-sort based on text offsets
|
|
// Look up each indirect value in appropriate scoring table and keep
|
|
// just the resulting langprobs
|
|
void LinearizeAll(ScoringContext* scoringcontext, bool score_cjk,
|
|
ScoringHitBuffer* hitbuffer) {
|
|
const CLD2TableSummary* base_obj; // unigram or quadgram
|
|
const CLD2TableSummary* base_obj2; // quadgram dual table
|
|
const CLD2TableSummary* delta_obj; // bigram or octagram
|
|
const CLD2TableSummary* distinct_obj; // bigram or octagram
|
|
uint16 base_hit;
|
|
if (score_cjk) {
|
|
base_obj = scoringcontext->scoringtables->unigram_compat_obj;
|
|
base_obj2 = scoringcontext->scoringtables->unigram_compat_obj;
|
|
delta_obj = scoringcontext->scoringtables->deltabi_obj;
|
|
distinct_obj = scoringcontext->scoringtables->distinctbi_obj;
|
|
base_hit = UNIHIT;
|
|
} else {
|
|
base_obj = scoringcontext->scoringtables->quadgram_obj;
|
|
base_obj2 = scoringcontext->scoringtables->quadgram_obj2;
|
|
delta_obj = scoringcontext->scoringtables->deltaocta_obj;
|
|
distinct_obj = scoringcontext->scoringtables->distinctocta_obj;
|
|
base_hit = QUADHIT;
|
|
}
|
|
|
|
int base_limit = hitbuffer->next_base;
|
|
int delta_limit = hitbuffer->next_delta;
|
|
int distinct_limit = hitbuffer->next_distinct;
|
|
int base_i = 0;
|
|
int delta_i = 0;
|
|
int distinct_i = 0;
|
|
int linear_i = 0;
|
|
|
|
// Start with an initial base hit for the default language for this script
|
|
// Inserting this avoids edge effects with no hits at all
|
|
hitbuffer->linear[linear_i].offset = hitbuffer->lowest_offset;
|
|
hitbuffer->linear[linear_i].type = base_hit;
|
|
hitbuffer->linear[linear_i].langprob =
|
|
DefaultLangProb(scoringcontext->ulscript);
|
|
++linear_i;
|
|
|
|
while ((base_i < base_limit) || (delta_i < delta_limit) ||
|
|
(distinct_i < distinct_limit)) {
|
|
int base_off = hitbuffer->base[base_i].offset;
|
|
int delta_off = hitbuffer->delta[delta_i].offset;
|
|
int distinct_off = hitbuffer->distinct[distinct_i].offset;
|
|
|
|
// Do delta and distinct first, so that they are not lost at base_limit
|
|
if ((delta_i < delta_limit) &&
|
|
(delta_off <= base_off) && (delta_off <= distinct_off)) {
|
|
// Add delta entry
|
|
int indirect = hitbuffer->delta[delta_i].indirect;
|
|
++delta_i;
|
|
uint32 langprob = delta_obj->kCLDTableInd[indirect];
|
|
if (langprob > 0) {
|
|
hitbuffer->linear[linear_i].offset = delta_off;
|
|
hitbuffer->linear[linear_i].type = DELTAHIT;
|
|
hitbuffer->linear[linear_i].langprob = langprob;
|
|
++linear_i;
|
|
}
|
|
}
|
|
else if ((distinct_i < distinct_limit) &&
|
|
(distinct_off <= base_off) && (distinct_off <= delta_off)) {
|
|
// Add distinct entry
|
|
int indirect = hitbuffer->distinct[distinct_i].indirect;
|
|
++distinct_i;
|
|
uint32 langprob = distinct_obj->kCLDTableInd[indirect];
|
|
if (langprob > 0) {
|
|
hitbuffer->linear[linear_i].offset = distinct_off;
|
|
hitbuffer->linear[linear_i].type = DISTINCTHIT;
|
|
hitbuffer->linear[linear_i].langprob = langprob;
|
|
++linear_i;
|
|
}
|
|
}
|
|
else {
|
|
// Add one or two base entries
|
|
int indirect = hitbuffer->base[base_i].indirect;
|
|
// First, get right scoring table
|
|
const CLD2TableSummary* local_base_obj = base_obj;
|
|
if ((indirect & 0x80000000u) != 0) {
|
|
local_base_obj = base_obj2;
|
|
indirect &= ~0x80000000u;
|
|
}
|
|
++base_i;
|
|
// One langprob in kQuadInd[0..SingleSize),
|
|
// two in kQuadInd[SingleSize..Size)
|
|
if (indirect < static_cast<int>(local_base_obj->kCLDTableSizeOne)) {
|
|
// Up to three languages at indirect
|
|
uint32 langprob = local_base_obj->kCLDTableInd[indirect];
|
|
if (langprob > 0) {
|
|
hitbuffer->linear[linear_i].offset = base_off;
|
|
hitbuffer->linear[linear_i].type = base_hit;
|
|
hitbuffer->linear[linear_i].langprob = langprob;
|
|
++linear_i;
|
|
}
|
|
} else {
|
|
// Up to six languages at start + 2 * (indirect - start)
|
|
indirect += (indirect - local_base_obj->kCLDTableSizeOne);
|
|
uint32 langprob = local_base_obj->kCLDTableInd[indirect];
|
|
uint32 langprob2 = local_base_obj->kCLDTableInd[indirect + 1];
|
|
if (langprob > 0) {
|
|
hitbuffer->linear[linear_i].offset = base_off;
|
|
hitbuffer->linear[linear_i].type = base_hit;
|
|
hitbuffer->linear[linear_i].langprob = langprob;
|
|
++linear_i;
|
|
}
|
|
if (langprob2 > 0) {
|
|
hitbuffer->linear[linear_i].offset = base_off;
|
|
hitbuffer->linear[linear_i].type = base_hit;
|
|
hitbuffer->linear[linear_i].langprob = langprob2;
|
|
++linear_i;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Update
|
|
hitbuffer->next_linear = linear_i;
|
|
|
|
// Add a dummy entry off the end, just to capture final offset
|
|
hitbuffer->linear[linear_i].offset =
|
|
hitbuffer->base[hitbuffer->next_base].offset;
|
|
hitbuffer->linear[linear_i].langprob = 0;
|
|
}
|
|
|
|
// Break linear array into chunks of ~20 quadgram hits or ~50 CJK unigram hits
|
|
void ChunkAll(int letter_offset, bool score_cjk, ScoringHitBuffer* hitbuffer) {
|
|
int chunksize;
|
|
uint16 base_hit;
|
|
if (score_cjk) {
|
|
chunksize = kChunksizeUnis;
|
|
base_hit = UNIHIT;
|
|
} else {
|
|
chunksize = kChunksizeQuads;
|
|
base_hit = QUADHIT;
|
|
}
|
|
|
|
int linear_i = 0;
|
|
int linear_off_end = hitbuffer->next_linear;
|
|
int text_i = letter_offset; // Next unseen text offset
|
|
int next_chunk_start = 0;
|
|
int bases_left = hitbuffer->next_base;
|
|
while (bases_left > 0) {
|
|
// Linearize one chunk
|
|
int base_len = chunksize; // Default; may be changed below
|
|
if (bases_left < (chunksize + (chunksize >> 1))) {
|
|
// If within 1.5 chunks of the end, avoid runts by using it all
|
|
base_len = bases_left;
|
|
} else if (bases_left < (2 * chunksize)) {
|
|
// Avoid runts by splitting 1.5 to 2 chunks in half (about 3/4 each)
|
|
base_len = (bases_left + 1) >> 1;
|
|
}
|
|
|
|
hitbuffer->chunk_start[next_chunk_start] = linear_i;
|
|
hitbuffer->chunk_offset[next_chunk_start] = text_i;
|
|
++next_chunk_start;
|
|
|
|
int base_count = 0;
|
|
while ((base_count < base_len) && (linear_i < linear_off_end)) {
|
|
if (hitbuffer->linear[linear_i].type == base_hit) {++base_count;}
|
|
++linear_i;
|
|
}
|
|
text_i = hitbuffer->linear[linear_i].offset; // Next unseen text offset
|
|
bases_left -= base_len;
|
|
}
|
|
|
|
// If no base hits at all, make a single dummy chunk
|
|
if (next_chunk_start == 0) {
|
|
hitbuffer->chunk_start[next_chunk_start] = 0;
|
|
hitbuffer->chunk_offset[next_chunk_start] = hitbuffer->linear[0].offset;
|
|
++next_chunk_start;
|
|
}
|
|
|
|
// Remember the linear array start of dummy entry
|
|
hitbuffer->next_chunk_start = next_chunk_start;
|
|
|
|
// Add a dummy entry off the end, just to capture final linear subscr
|
|
hitbuffer->chunk_start[next_chunk_start] = hitbuffer->next_linear;
|
|
hitbuffer->chunk_offset[next_chunk_start] = text_i;
|
|
}
|
|
|
|
|
|
// Merge-sort the individual hit arrays, go indirect on the scoring subscripts,
|
|
// break linear array into chunks.
|
|
//
|
|
// Input:
|
|
// hitbuffer base, delta, distinct arrays
|
|
// Output:
|
|
// linear array
|
|
// chunk_start array
|
|
//
|
|
void LinearizeHitBuffer(int letter_offset,
|
|
ScoringContext* scoringcontext,
|
|
bool more_to_come, bool score_cjk,
|
|
ScoringHitBuffer* hitbuffer) {
|
|
LinearizeAll(scoringcontext, score_cjk, hitbuffer);
|
|
ChunkAll(letter_offset, score_cjk, hitbuffer);
|
|
}
|
|
|
|
|
|
|
|
// The hitbuffer is in an awkward form -- three sets of base/delta/distinct
|
|
// scores, each with an indirect subscript to one of six scoring tables, some
|
|
// of which can yield two langprobs for six languages, others one langprob for
|
|
// three languages. The only correlation between base/delta/distinct is their
|
|
// offsets into the letters-only text buffer.
|
|
//
|
|
// SummaryBuffer needs to be built to linear, giving linear offset of start of
|
|
// each chunk
|
|
//
|
|
// So we first do all the langprob lookups and merge-sort by offset to make
|
|
// a single linear vector, building a side vector of chunk beginnings as we go.
|
|
// The sharpening is simply moving the beginnings, scoring is a simple linear
|
|
// sweep, etc.
|
|
|
|
void ProcessHitBuffer(const LangSpan& scriptspan,
|
|
int letter_offset,
|
|
ScoringContext* scoringcontext,
|
|
DocTote* doc_tote,
|
|
ResultChunkVector* vec,
|
|
bool more_to_come, bool score_cjk,
|
|
ScoringHitBuffer* hitbuffer) {
|
|
if (scoringcontext->flags_cld2_verbose) {
|
|
fprintf(scoringcontext->debug_file, "Hitbuffer[) ");
|
|
DumpHitBuffer(scoringcontext->debug_file, scriptspan.text, hitbuffer);
|
|
}
|
|
|
|
LinearizeHitBuffer(letter_offset, scoringcontext, more_to_come, score_cjk,
|
|
hitbuffer);
|
|
|
|
if (scoringcontext->flags_cld2_verbose) {
|
|
fprintf(scoringcontext->debug_file, "Linear[) ");
|
|
DumpLinearBuffer(scoringcontext->debug_file, scriptspan.text, hitbuffer);
|
|
}
|
|
|
|
SummaryBuffer summarybuffer;
|
|
summarybuffer.n = 0;
|
|
ChunkSpan last_cspan;
|
|
ScoreAllHits(scriptspan.text, scriptspan.ulscript,
|
|
more_to_come, score_cjk, hitbuffer,
|
|
scoringcontext,
|
|
&summarybuffer, &last_cspan);
|
|
|
|
if (scoringcontext->flags_cld2_verbose) {
|
|
DumpSummaryBuffer(scoringcontext->debug_file, &summarybuffer);
|
|
}
|
|
|
|
if (vec != NULL) {
|
|
// Sharpen boundaries of summarybuffer
|
|
// This is not a high-performance path
|
|
SharpenBoundaries(scriptspan.text, more_to_come, hitbuffer, scoringcontext,
|
|
&summarybuffer);
|
|
// Show after the sharpening
|
|
// CLD2_Debug2(scriptspan.text, more_to_come, score_cjk,
|
|
// hitbuffer, scoringcontext, &summarybuffer);
|
|
|
|
if (scoringcontext->flags_cld2_verbose) {
|
|
DumpSummaryBuffer(scoringcontext->debug_file, &summarybuffer);
|
|
}
|
|
}
|
|
|
|
SummaryBufferToDocTote(&summarybuffer, more_to_come, doc_tote);
|
|
SummaryBufferToVector(scoringcontext->scanner, scriptspan.text,
|
|
&summarybuffer, more_to_come, vec);
|
|
}
|
|
|
|
void SpliceHitBuffer(ScoringHitBuffer* hitbuffer, int next_offset) {
|
|
// Splice hitbuffer and summarybuffer for next round. With big chunks and
|
|
// distinctive-word state carried across chunks, we might not need to do this.
|
|
hitbuffer->next_base = 0;
|
|
hitbuffer->next_delta = 0;
|
|
hitbuffer->next_distinct = 0;
|
|
hitbuffer->next_linear = 0;
|
|
hitbuffer->next_chunk_start = 0;
|
|
hitbuffer->lowest_offset = next_offset;
|
|
}
|
|
|
|
|
|
// Score RTypeNone or RTypeOne scriptspan into doc_tote and vec, updating
|
|
// scoringcontext
|
|
void ScoreEntireScriptSpan(const LangSpan& scriptspan,
|
|
ScoringContext* scoringcontext,
|
|
DocTote* doc_tote,
|
|
ResultChunkVector* vec) {
|
|
int bytes = scriptspan.text_bytes;
|
|
// Artificially set score to 1024 per 1KB, or 1 per byte
|
|
int score = bytes;
|
|
int reliability = 100;
|
|
// doc_tote uses full languages
|
|
Language one_one_lang = DefaultLanguage(scriptspan.ulscript);
|
|
doc_tote->Add(one_one_lang, bytes, score, reliability);
|
|
|
|
if (scoringcontext->flags_cld2_html) {
|
|
ChunkSummary chunksummary = {
|
|
1, 0,
|
|
one_one_lang, UNKNOWN_LANGUAGE, score, 1,
|
|
bytes, 0, scriptspan.ulscript, reliability, reliability
|
|
};
|
|
CLD2_Debug(scriptspan.text, 1, scriptspan.text_bytes,
|
|
false, false, NULL,
|
|
scoringcontext, NULL, &chunksummary);
|
|
}
|
|
|
|
// First byte is always a space
|
|
JustOneItemToVector(scoringcontext->scanner, scriptspan.text,
|
|
one_one_lang, 1, bytes - 1, vec);
|
|
|
|
scoringcontext->prior_chunk_lang = UNKNOWN_LANGUAGE;
|
|
}
|
|
|
|
// Score RTypeCJK scriptspan into doc_tote and vec, updating scoringcontext
|
|
void ScoreCJKScriptSpan(const LangSpan& scriptspan,
|
|
ScoringContext* scoringcontext,
|
|
DocTote* doc_tote,
|
|
ResultChunkVector* vec) {
|
|
// Allocate three parallel arrays of scoring hits
|
|
ScoringHitBuffer* hitbuffer = new ScoringHitBuffer;
|
|
hitbuffer->init();
|
|
hitbuffer->ulscript = scriptspan.ulscript;
|
|
|
|
scoringcontext->prior_chunk_lang = UNKNOWN_LANGUAGE;
|
|
scoringcontext->oldest_distinct_boost = 0;
|
|
|
|
// Incoming scriptspan has a single leading space at scriptspan.text[0]
|
|
// and three trailing spaces then NUL at scriptspan.text[text_bytes + 0/1/2/3]
|
|
|
|
int letter_offset = 1; // Skip initial space
|
|
hitbuffer->lowest_offset = letter_offset;
|
|
int letter_limit = scriptspan.text_bytes;
|
|
while (letter_offset < letter_limit) {
|
|
if (scoringcontext->flags_cld2_verbose) {
|
|
fprintf(scoringcontext->debug_file, " ScoreCJKScriptSpan[%d,%d)<br>\n",
|
|
letter_offset, letter_limit);
|
|
}
|
|
//
|
|
// Fill up one hitbuffer, possibly splicing onto previous fragment
|
|
//
|
|
// NOTE: GetUniHits deals with close repeats
|
|
// NOTE: After last chunk there is always a hitbuffer entry with an offset
|
|
// just off the end of the text = next_offset.
|
|
int next_offset = GetUniHits(scriptspan.text, letter_offset, letter_limit,
|
|
scoringcontext, hitbuffer);
|
|
// NOTE: GetBiHitVectors deals with close repeats,
|
|
// does one hash and two lookups (delta and distinct) per word
|
|
GetBiHits(scriptspan.text, letter_offset, next_offset,
|
|
scoringcontext, hitbuffer);
|
|
|
|
//
|
|
// Score one hitbuffer in chunks to summarybuffer
|
|
//
|
|
bool more_to_come = next_offset < letter_limit;
|
|
bool score_cjk = true;
|
|
ProcessHitBuffer(scriptspan, letter_offset, scoringcontext, doc_tote, vec,
|
|
more_to_come, score_cjk, hitbuffer);
|
|
SpliceHitBuffer(hitbuffer, next_offset);
|
|
|
|
letter_offset = next_offset;
|
|
}
|
|
|
|
delete hitbuffer;
|
|
// Context across buffers is not connected yet
|
|
scoringcontext->prior_chunk_lang = UNKNOWN_LANGUAGE;
|
|
}
|
|
|
|
|
|
|
|
// Score RTypeMany scriptspan into doc_tote and vec, updating scoringcontext
|
|
// We have a scriptspan with all lowercase text in one script. Look up
|
|
// quadgrams and octagrams, saving the hits in three parallel vectors.
|
|
// Score from those vectors in chunks, toting each chunk to get a single
|
|
// language, and combining into the overall document score. The hit vectors
|
|
// in general are not big enough to handle and entire scriptspan, so
|
|
// repeat until the entire scriptspan is scored.
|
|
// Caller deals with minimizing numbr of runt scriptspans
|
|
// This routine deals with minimizing number of runt chunks.
|
|
//
|
|
// Returns updated scoringcontext
|
|
// Returns updated doc_tote
|
|
// If vec != NULL, appends to that vector of ResultChunk's
|
|
void ScoreQuadScriptSpan(const LangSpan& scriptspan,
|
|
ScoringContext* scoringcontext,
|
|
DocTote* doc_tote,
|
|
ResultChunkVector* vec) {
|
|
// Allocate three parallel arrays of scoring hits
|
|
ScoringHitBuffer* hitbuffer = new ScoringHitBuffer;
|
|
hitbuffer->init();
|
|
hitbuffer->ulscript = scriptspan.ulscript;
|
|
|
|
scoringcontext->prior_chunk_lang = UNKNOWN_LANGUAGE;
|
|
scoringcontext->oldest_distinct_boost = 0;
|
|
|
|
// Incoming scriptspan has a single leading space at scriptspan.text[0]
|
|
// and three trailing spaces then NUL at scriptspan.text[text_bytes + 0/1/2/3]
|
|
|
|
int letter_offset = 1; // Skip initial space
|
|
hitbuffer->lowest_offset = letter_offset;
|
|
int letter_limit = scriptspan.text_bytes;
|
|
while (letter_offset < letter_limit) {
|
|
//
|
|
// Fill up one hitbuffer, possibly splicing onto previous fragment
|
|
//
|
|
// NOTE: GetQuadHits deals with close repeats
|
|
// NOTE: After last chunk there is always a hitbuffer entry with an offset
|
|
// just off the end of the text = next_offset.
|
|
int next_offset = GetQuadHits(scriptspan.text, letter_offset, letter_limit,
|
|
scoringcontext, hitbuffer);
|
|
// If true, there is more text to process in this scriptspan
|
|
// NOTE: GetOctaHitVectors deals with close repeats,
|
|
// does one hash and two lookups (delta and distinct) per word
|
|
GetOctaHits(scriptspan.text, letter_offset, next_offset,
|
|
scoringcontext, hitbuffer);
|
|
|
|
//
|
|
// Score one hitbuffer in chunks to summarybuffer
|
|
//
|
|
bool more_to_come = next_offset < letter_limit;
|
|
bool score_cjk = false;
|
|
ProcessHitBuffer(scriptspan, letter_offset, scoringcontext, doc_tote, vec,
|
|
more_to_come, score_cjk, hitbuffer);
|
|
SpliceHitBuffer(hitbuffer, next_offset);
|
|
|
|
letter_offset = next_offset;
|
|
}
|
|
|
|
delete hitbuffer;
|
|
}
|
|
|
|
|
|
// Score one scriptspan into doc_tote and vec, updating scoringcontext
|
|
// Inputs:
|
|
// One scriptspan of perhaps 40-60KB, all same script lower-case letters
|
|
// and single ASCII spaces. First character is a space to allow simple
|
|
// begining-of-word detect. End of buffer has three spaces and NUL to
|
|
// allow easy scan-to-end-of-word.
|
|
// Scoring context of
|
|
// scoring tables
|
|
// flags
|
|
// running boosts
|
|
// Outputs:
|
|
// Updated doc_tote giving overall languages and byte counts
|
|
// Optional updated chunk vector giving offset, length, language
|
|
//
|
|
// Caller initializes flags, boosts, doc_tote and vec.
|
|
// Caller aggregates across multiple scriptspans
|
|
// Caller calculates final document result
|
|
// Caller deals with detecting and triggering suppression of repeated text.
|
|
//
|
|
// This top-level routine just chooses the recognition type and calls one of
|
|
// the next-level-down routines.
|
|
//
|
|
void ScoreOneScriptSpan(const LangSpan& scriptspan,
|
|
ScoringContext* scoringcontext,
|
|
DocTote* doc_tote,
|
|
ResultChunkVector* vec) {
|
|
if (scoringcontext->flags_cld2_verbose) {
|
|
fprintf(scoringcontext->debug_file, "<br>ScoreOneScriptSpan(%s,%d) ",
|
|
ULScriptCode(scriptspan.ulscript), scriptspan.text_bytes);
|
|
// Optionally print the chunk lowercase letters/marks text
|
|
string temp(&scriptspan.text[0], scriptspan.text_bytes);
|
|
fprintf(scoringcontext->debug_file, "'%s'",
|
|
GetHtmlEscapedText(temp).c_str());
|
|
fprintf(scoringcontext->debug_file, "<br>\n");
|
|
}
|
|
scoringcontext->prior_chunk_lang = UNKNOWN_LANGUAGE;
|
|
scoringcontext->oldest_distinct_boost = 0;
|
|
ULScriptRType rtype = ULScriptRecognitionType(scriptspan.ulscript);
|
|
if (scoringcontext->flags_cld2_score_as_quads && (rtype != RTypeCJK)) {
|
|
rtype = RTypeMany;
|
|
}
|
|
switch (rtype) {
|
|
case RTypeNone:
|
|
case RTypeOne:
|
|
ScoreEntireScriptSpan(scriptspan, scoringcontext, doc_tote, vec);
|
|
break;
|
|
case RTypeCJK:
|
|
ScoreCJKScriptSpan(scriptspan, scoringcontext, doc_tote, vec);
|
|
break;
|
|
case RTypeMany:
|
|
ScoreQuadScriptSpan(scriptspan, scoringcontext, doc_tote, vec);
|
|
break;
|
|
}
|
|
}
|
|
|
|
} // End namespace CLD2
|
|
|