26 #if defined(_WIN32) && !defined(__GNUC__)
27 #define strtok_r(str, delim, saveptr) strtok_s(str, delim, saveptr)
32 static const char kAmbigDelimiters[] =
"\t ";
33 static const char kIllegalMsg[] =
34 "Illegal ambiguity specification on line %d\n";
35 static const char kIllegalUnicharMsg[] =
36 "Illegal unichar %s in ambiguity specification\n";
55 bool use_ambigs_for_adaption) {
56 for (
int i = 0; i < unicharset.size(); ++i) {
57 replace_ambigs_.push_back(
nullptr);
58 dang_ambigs_.push_back(
nullptr);
59 one_to_one_definite_ambigs_.push_back(
nullptr);
60 if (use_ambigs_for_adaption) {
61 ambigs_for_adaption_.push_back(
nullptr);
62 reverse_ambigs_for_adaption_.push_back(
nullptr);
78 bool use_ambigs_for_adaption,
82 if (debug_level)
tprintf(
"Reading ambiguities\n");
84 int test_ambig_part_size;
85 int replacement_ambig_part_size;
89 char *buffer =
new char[kBufferSize];
100 version = static_cast<int>(strtol(buffer+1,
nullptr, 10));
105 while (ambig_file->
FGets(buffer, kBufferSize) !=
nullptr) {
107 if (debug_level > 2)
tprintf(
"read line %s\n", buffer);
109 if (!ParseAmbiguityLine(line_num, version, debug_level, encoder_set,
110 buffer, &test_ambig_part_size, test_unichar_ids,
111 &replacement_ambig_part_size,
112 replacement_string, &
type))
continue;
117 test_ambig_part_size, test_unichar_ids,
118 replacement_ambig_part_size, replacement_string,
type,
119 ambig_spec, unicharset))
123 if (test_ambig_part_size == 1 &&
125 if (one_to_one_definite_ambigs_[test_unichar_ids[0]] ==
nullptr) {
126 one_to_one_definite_ambigs_[test_unichar_ids[0]] =
new UnicharIdVector();
128 one_to_one_definite_ambigs_[test_unichar_ids[0]]->push_back(
129 ambig_spec->correct_ngram_id);
132 if (use_ambigs_for_adaption) {
136 if (unicharset->
encode_string(replacement_string,
true, &encoding,
138 for (i = 0; i < test_ambig_part_size; ++i) {
139 if (ambigs_for_adaption_[test_unichar_ids[i]] ==
nullptr) {
142 adaption_ambigs_entry = ambigs_for_adaption_[test_unichar_ids[i]];
143 for (
int r = 0; r < encoding.
size(); ++r) {
148 for (j = 0; j < adaption_ambigs_entry->
size() &&
149 (*adaption_ambigs_entry)[j] > id_to_insert; ++j);
150 if (j < adaption_ambigs_entry->size()) {
151 if ((*adaption_ambigs_entry)[j] != id_to_insert) {
152 adaption_ambigs_entry->
insert(id_to_insert, j);
155 adaption_ambigs_entry->
push_back(id_to_insert);
165 if (use_ambigs_for_adaption) {
166 for (i = 0; i < ambigs_for_adaption_.size(); ++i) {
167 adaption_ambigs_entry = ambigs_for_adaption_[i];
168 if (adaption_ambigs_entry ==
nullptr)
continue;
169 for (j = 0; j < adaption_ambigs_entry->
size(); ++j) {
170 UNICHAR_ID ambig_id = (*adaption_ambigs_entry)[j];
171 if (reverse_ambigs_for_adaption_[ambig_id] ==
nullptr) {
174 reverse_ambigs_for_adaption_[ambig_id]->push_back(i);
180 if (debug_level > 1) {
181 for (
int tbl = 0; tbl < 2; ++tbl) {
183 (tbl == 0) ? replace_ambigs_ : dang_ambigs_;
184 for (i = 0; i < print_table.
size(); ++i) {
185 AmbigSpec_LIST *lst = print_table[i];
186 if (lst ==
nullptr)
continue;
188 tprintf(
"%s Ambiguities for %s:\n",
189 (tbl == 0) ?
"Replaceable" :
"Dangerous",
192 AmbigSpec_IT lst_it(lst);
193 for (lst_it.mark_cycle_pt(); !lst_it.cycled_list(); lst_it.forward()) {
202 if (use_ambigs_for_adaption) {
203 for (
int vec_id = 0; vec_id < 2; ++vec_id) {
205 ambigs_for_adaption_ : reverse_ambigs_for_adaption_;
206 for (i = 0; i < vec.
size(); ++i) {
207 adaption_ambigs_entry = vec[i];
208 if (adaption_ambigs_entry !=
nullptr) {
209 tprintf(
"%sAmbigs for adaption for %s:\n",
210 (vec_id == 0) ?
"" :
"Reverse ",
212 for (j = 0; j < adaption_ambigs_entry->
size(); ++j) {
214 (*adaption_ambigs_entry)[j]).
c_str());
224 bool UnicharAmbigs::ParseAmbiguityLine(
225 int line_num,
int version,
int debug_level,
const UNICHARSET &unicharset,
226 char *buffer,
int *test_ambig_part_size,
UNICHAR_ID *test_unichar_ids,
227 int *replacement_ambig_part_size,
char *replacement_string,
int *
type) {
232 input.split(
' ', &fields);
233 if (fields.
size() != 3) {
234 if (debug_level)
tprintf(kIllegalMsg, line_num);
239 if (!unicharset.
encode_string(fields[0].c_str(),
true, &unichars,
nullptr,
243 *test_ambig_part_size = unichars.
size();
246 tprintf(
"Too many unichars in ambiguity on line %d\n", line_num);
250 for (
int i = 0; i < unichars.
size(); ++i)
251 test_unichar_ids[i] = unichars[i];
252 test_unichar_ids[unichars.
size()] = INVALID_UNICHAR_ID;
254 if (!unicharset.
encode_string(fields[1].c_str(),
true, &unichars,
nullptr,
258 *replacement_ambig_part_size = unichars.
size();
261 tprintf(
"Too many unichars in ambiguity on line %d\n", line_num);
264 if (sscanf(fields[2].c_str(),
"%d",
type) != 1) {
265 if (debug_level)
tprintf(kIllegalMsg, line_num);
274 if (!(token = strtok_r(buffer, kAmbigDelimiters, &next_token)) ||
275 !sscanf(token,
"%d", test_ambig_part_size) ||
276 *test_ambig_part_size <= 0) {
277 if (debug_level)
tprintf(kIllegalMsg, line_num);
282 tprintf(
"Too many unichars in ambiguity on line %d\n", line_num);
285 for (i = 0; i < *test_ambig_part_size; ++i) {
286 if (!(token = strtok_r(
nullptr, kAmbigDelimiters, &next_token)))
break;
288 if (debug_level)
tprintf(kIllegalUnicharMsg, token);
293 test_unichar_ids[i] = INVALID_UNICHAR_ID;
295 if (i != *test_ambig_part_size ||
296 !(token = strtok_r(
nullptr, kAmbigDelimiters, &next_token)) ||
297 !sscanf(token,
"%d", replacement_ambig_part_size) ||
298 *replacement_ambig_part_size <= 0) {
299 if (debug_level)
tprintf(kIllegalMsg, line_num);
304 tprintf(
"Too many unichars in ambiguity on line %d\n", line_num);
307 replacement_string[0] =
'\0';
308 for (i = 0; i < *replacement_ambig_part_size; ++i) {
309 if (!(token = strtok_r(
nullptr, kAmbigDelimiters, &next_token)))
break;
310 strcat(replacement_string, token);
312 if (debug_level)
tprintf(kIllegalUnicharMsg, token);
316 if (i != *replacement_ambig_part_size) {
317 if (debug_level)
tprintf(kIllegalMsg, line_num);
330 if (!(token = strtok_r(
nullptr, kAmbigDelimiters, &next_token)) ||
331 !sscanf(token,
"%d",
type)) {
332 if (debug_level)
tprintf(kIllegalMsg, line_num);
339 bool UnicharAmbigs::InsertIntoTable(
341 UNICHAR_ID *test_unichar_ids,
int replacement_ambig_part_size,
342 const char *replacement_string,
int type,
343 AmbigSpec *ambig_spec,
UNICHARSET *unicharset) {
344 ambig_spec->type = static_cast<AmbigType>(
type);
345 if (test_ambig_part_size == 1 && replacement_ambig_part_size == 1 &&
346 unicharset->
to_lower(test_unichar_ids[0]) ==
351 ambig_spec->wrong_ngram_size =
366 ambig_spec->correct_ngram_id =
368 if (replacement_ambig_part_size > 1) {
369 unicharset->
set_isngram(ambig_spec->correct_ngram_id,
true);
373 for (i = 0; i < test_ambig_part_size; ++i) {
375 if (test_ambig_part_size == 1) {
376 unichar_id = ambig_spec->correct_ngram_id;
379 replacement_string, i, test_ambig_part_size,
false);
383 ambig_spec->correct_fragments[i] = unichar_id;
385 ambig_spec->correct_fragments[i] = INVALID_UNICHAR_ID;
389 if (table[test_unichar_ids[0]] ==
nullptr) {
390 table[test_unichar_ids[0]] =
new AmbigSpec_LIST();
392 if (table[test_unichar_ids[0]]->add_sorted(