tesseract  5.0.0-alpha-619-ge9db
ambigs.cpp
Go to the documentation of this file.
1 // File: ambigs.cpp
3 // Description: Functions for dealing with ambiguities
4 // (training and recognition).
5 // Author: Daria Antonova
6 //
7 // (C) Copyright 2008, Google Inc.
8 // Licensed under the Apache License, Version 2.0 (the "License");
9 // you may not use this file except in compliance with the License.
10 // You may obtain a copy of the License at
11 // http://www.apache.org/licenses/LICENSE-2.0
12 // Unless required by applicable law or agreed to in writing, software
13 // distributed under the License is distributed on an "AS IS" BASIS,
14 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 // See the License for the specific language governing permissions and
16 // limitations under the License.
17 //
19 
20 #include "ambigs.h"
21 
22 #include <cstdio>
23 #include <tesseract/helpers.h>
24 #include "universalambigs.h"
25 
26 #if defined(_WIN32) && !defined(__GNUC__)
27 #define strtok_r(str, delim, saveptr) strtok_s(str, delim, saveptr)
28 #endif /* _WIN32 && !__GNUC__ */
29 
30 namespace tesseract {
31 
32 static const char kAmbigDelimiters[] = "\t ";
33 static const char kIllegalMsg[] =
34  "Illegal ambiguity specification on line %d\n";
35 static const char kIllegalUnicharMsg[] =
36  "Illegal unichar %s in ambiguity specification\n";
37 
38 // Maximum line size:
39 // 10 for sizes of ambigs, tabs, abmig type and newline
40 // UNICHAR_LEN * (MAX_AMBIG_SIZE + 1) for each part of the ambig
42 
44  wrong_ngram[0] = INVALID_UNICHAR_ID;
45  correct_fragments[0] = INVALID_UNICHAR_ID;
46  correct_ngram_id = INVALID_UNICHAR_ID;
47  type = NOT_AMBIG;
48  wrong_ngram_size = 0;
49 }
50 
52 
53 // Initializes the ambigs by adding a nullptr pointer to each table.
54 void UnicharAmbigs::InitUnicharAmbigs(const UNICHARSET& unicharset,
55  bool use_ambigs_for_adaption) {
56  for (int i = 0; i < unicharset.size(); ++i) {
57  replace_ambigs_.push_back(nullptr);
58  dang_ambigs_.push_back(nullptr);
59  one_to_one_definite_ambigs_.push_back(nullptr);
60  if (use_ambigs_for_adaption) {
61  ambigs_for_adaption_.push_back(nullptr);
62  reverse_ambigs_for_adaption_.push_back(nullptr);
63  }
64  }
65 }
66 
67 // Loads the universal ambigs that are useful for any language.
68 void UnicharAmbigs::LoadUniversal(const UNICHARSET& encoder_set,
69  UNICHARSET* unicharset) {
70  TFile file;
72  LoadUnicharAmbigs(encoder_set, &file, 0, false, unicharset);
73 }
74 
76  TFile *ambig_file,
77  int debug_level,
78  bool use_ambigs_for_adaption,
79  UNICHARSET *unicharset) {
80  int i, j;
81  UnicharIdVector *adaption_ambigs_entry;
82  if (debug_level) tprintf("Reading ambiguities\n");
83 
84  int test_ambig_part_size;
85  int replacement_ambig_part_size;
86  // The space for buffer is allocated on the heap to avoid
87  // GCC frame size warning.
88  const int kBufferSize = 10 + 2 * kMaxAmbigStringSize;
89  char *buffer = new char[kBufferSize];
90  char replacement_string[kMaxAmbigStringSize];
91  UNICHAR_ID test_unichar_ids[MAX_AMBIG_SIZE + 1];
92  int line_num = 0;
93  int type = NOT_AMBIG;
94 
95  // Determine the version of the ambigs file.
96  int version = 0;
97  ASSERT_HOST(ambig_file->FGets(buffer, kBufferSize) != nullptr &&
98  strlen(buffer) > 0);
99  if (*buffer == 'v') {
100  version = static_cast<int>(strtol(buffer+1, nullptr, 10));
101  ++line_num;
102  } else {
103  ambig_file->Rewind();
104  }
105  while (ambig_file->FGets(buffer, kBufferSize) != nullptr) {
106  chomp_string(buffer);
107  if (debug_level > 2) tprintf("read line %s\n", buffer);
108  ++line_num;
109  if (!ParseAmbiguityLine(line_num, version, debug_level, encoder_set,
110  buffer, &test_ambig_part_size, test_unichar_ids,
111  &replacement_ambig_part_size,
112  replacement_string, &type)) continue;
113  // Construct AmbigSpec and add it to the appropriate AmbigSpec_LIST.
114  auto *ambig_spec = new AmbigSpec();
115  if (!InsertIntoTable((type == REPLACE_AMBIG) ? replace_ambigs_
116  : dang_ambigs_,
117  test_ambig_part_size, test_unichar_ids,
118  replacement_ambig_part_size, replacement_string, type,
119  ambig_spec, unicharset))
120  continue;
121 
122  // Update one_to_one_definite_ambigs_.
123  if (test_ambig_part_size == 1 &&
124  replacement_ambig_part_size == 1 && type == DEFINITE_AMBIG) {
125  if (one_to_one_definite_ambigs_[test_unichar_ids[0]] == nullptr) {
126  one_to_one_definite_ambigs_[test_unichar_ids[0]] = new UnicharIdVector();
127  }
128  one_to_one_definite_ambigs_[test_unichar_ids[0]]->push_back(
129  ambig_spec->correct_ngram_id);
130  }
131  // Update ambigs_for_adaption_.
132  if (use_ambigs_for_adaption) {
133  GenericVector<UNICHAR_ID> encoding;
134  // Silently ignore invalid strings, as before, so it is safe to use a
135  // universal ambigs file.
136  if (unicharset->encode_string(replacement_string, true, &encoding,
137  nullptr, nullptr)) {
138  for (i = 0; i < test_ambig_part_size; ++i) {
139  if (ambigs_for_adaption_[test_unichar_ids[i]] == nullptr) {
140  ambigs_for_adaption_[test_unichar_ids[i]] = new UnicharIdVector();
141  }
142  adaption_ambigs_entry = ambigs_for_adaption_[test_unichar_ids[i]];
143  for (int r = 0; r < encoding.size(); ++r) {
144  UNICHAR_ID id_to_insert = encoding[r];
145  ASSERT_HOST(id_to_insert != INVALID_UNICHAR_ID);
146  // Add the new unichar id to adaption_ambigs_entry (only if the
147  // vector does not already contain it) keeping it in sorted order.
148  for (j = 0; j < adaption_ambigs_entry->size() &&
149  (*adaption_ambigs_entry)[j] > id_to_insert; ++j);
150  if (j < adaption_ambigs_entry->size()) {
151  if ((*adaption_ambigs_entry)[j] != id_to_insert) {
152  adaption_ambigs_entry->insert(id_to_insert, j);
153  }
154  } else {
155  adaption_ambigs_entry->push_back(id_to_insert);
156  }
157  }
158  }
159  }
160  }
161  }
162  delete[] buffer;
163 
164  // Fill in reverse_ambigs_for_adaption from ambigs_for_adaption vector.
165  if (use_ambigs_for_adaption) {
166  for (i = 0; i < ambigs_for_adaption_.size(); ++i) {
167  adaption_ambigs_entry = ambigs_for_adaption_[i];
168  if (adaption_ambigs_entry == nullptr) continue;
169  for (j = 0; j < adaption_ambigs_entry->size(); ++j) {
170  UNICHAR_ID ambig_id = (*adaption_ambigs_entry)[j];
171  if (reverse_ambigs_for_adaption_[ambig_id] == nullptr) {
172  reverse_ambigs_for_adaption_[ambig_id] = new UnicharIdVector();
173  }
174  reverse_ambigs_for_adaption_[ambig_id]->push_back(i);
175  }
176  }
177  }
178 
179  // Print what was read from the input file.
180  if (debug_level > 1) {
181  for (int tbl = 0; tbl < 2; ++tbl) {
182  const UnicharAmbigsVector &print_table =
183  (tbl == 0) ? replace_ambigs_ : dang_ambigs_;
184  for (i = 0; i < print_table.size(); ++i) {
185  AmbigSpec_LIST *lst = print_table[i];
186  if (lst == nullptr) continue;
187  if (!lst->empty()) {
188  tprintf("%s Ambiguities for %s:\n",
189  (tbl == 0) ? "Replaceable" : "Dangerous",
190  unicharset->debug_str(i).c_str());
191  }
192  AmbigSpec_IT lst_it(lst);
193  for (lst_it.mark_cycle_pt(); !lst_it.cycled_list(); lst_it.forward()) {
194  AmbigSpec *ambig_spec = lst_it.data();
195  tprintf("wrong_ngram:");
196  UnicharIdArrayUtils::print(ambig_spec->wrong_ngram, *unicharset);
197  tprintf("correct_fragments:");
198  UnicharIdArrayUtils::print(ambig_spec->correct_fragments, *unicharset);
199  }
200  }
201  }
202  if (use_ambigs_for_adaption) {
203  for (int vec_id = 0; vec_id < 2; ++vec_id) {
204  const GenericVector<UnicharIdVector *> &vec = (vec_id == 0) ?
205  ambigs_for_adaption_ : reverse_ambigs_for_adaption_;
206  for (i = 0; i < vec.size(); ++i) {
207  adaption_ambigs_entry = vec[i];
208  if (adaption_ambigs_entry != nullptr) {
209  tprintf("%sAmbigs for adaption for %s:\n",
210  (vec_id == 0) ? "" : "Reverse ",
211  unicharset->debug_str(i).c_str());
212  for (j = 0; j < adaption_ambigs_entry->size(); ++j) {
213  tprintf("%s ", unicharset->debug_str(
214  (*adaption_ambigs_entry)[j]).c_str());
215  }
216  tprintf("\n");
217  }
218  }
219  }
220  }
221  }
222 }
223 
224 bool UnicharAmbigs::ParseAmbiguityLine(
225  int line_num, int version, int debug_level, const UNICHARSET &unicharset,
226  char *buffer, int *test_ambig_part_size, UNICHAR_ID *test_unichar_ids,
227  int *replacement_ambig_part_size, char *replacement_string, int *type) {
228  if (version > 1) {
229  // Simpler format is just wrong-string correct-string type\n.
230  STRING input(buffer);
231  GenericVector<STRING> fields;
232  input.split(' ', &fields);
233  if (fields.size() != 3) {
234  if (debug_level) tprintf(kIllegalMsg, line_num);
235  return false;
236  }
237  // Encode wrong-string.
238  GenericVector<UNICHAR_ID> unichars;
239  if (!unicharset.encode_string(fields[0].c_str(), true, &unichars, nullptr,
240  nullptr)) {
241  return false;
242  }
243  *test_ambig_part_size = unichars.size();
244  if (*test_ambig_part_size > MAX_AMBIG_SIZE) {
245  if (debug_level)
246  tprintf("Too many unichars in ambiguity on line %d\n", line_num);
247  return false;
248  }
249  // Copy encoded string to output.
250  for (int i = 0; i < unichars.size(); ++i)
251  test_unichar_ids[i] = unichars[i];
252  test_unichar_ids[unichars.size()] = INVALID_UNICHAR_ID;
253  // Encode replacement-string to check validity.
254  if (!unicharset.encode_string(fields[1].c_str(), true, &unichars, nullptr,
255  nullptr)) {
256  return false;
257  }
258  *replacement_ambig_part_size = unichars.size();
259  if (*replacement_ambig_part_size > MAX_AMBIG_SIZE) {
260  if (debug_level)
261  tprintf("Too many unichars in ambiguity on line %d\n", line_num);
262  return false;
263  }
264  if (sscanf(fields[2].c_str(), "%d", type) != 1) {
265  if (debug_level) tprintf(kIllegalMsg, line_num);
266  return false;
267  }
268  snprintf(replacement_string, kMaxAmbigStringSize, "%s", fields[1].c_str());
269  return true;
270  }
271  int i;
272  char *token;
273  char *next_token;
274  if (!(token = strtok_r(buffer, kAmbigDelimiters, &next_token)) ||
275  !sscanf(token, "%d", test_ambig_part_size) ||
276  *test_ambig_part_size <= 0) {
277  if (debug_level) tprintf(kIllegalMsg, line_num);
278  return false;
279  }
280  if (*test_ambig_part_size > MAX_AMBIG_SIZE) {
281  if (debug_level)
282  tprintf("Too many unichars in ambiguity on line %d\n", line_num);
283  return false;
284  }
285  for (i = 0; i < *test_ambig_part_size; ++i) {
286  if (!(token = strtok_r(nullptr, kAmbigDelimiters, &next_token))) break;
287  if (!unicharset.contains_unichar(token)) {
288  if (debug_level) tprintf(kIllegalUnicharMsg, token);
289  break;
290  }
291  test_unichar_ids[i] = unicharset.unichar_to_id(token);
292  }
293  test_unichar_ids[i] = INVALID_UNICHAR_ID;
294 
295  if (i != *test_ambig_part_size ||
296  !(token = strtok_r(nullptr, kAmbigDelimiters, &next_token)) ||
297  !sscanf(token, "%d", replacement_ambig_part_size) ||
298  *replacement_ambig_part_size <= 0) {
299  if (debug_level) tprintf(kIllegalMsg, line_num);
300  return false;
301  }
302  if (*replacement_ambig_part_size > MAX_AMBIG_SIZE) {
303  if (debug_level)
304  tprintf("Too many unichars in ambiguity on line %d\n", line_num);
305  return false;
306  }
307  replacement_string[0] = '\0';
308  for (i = 0; i < *replacement_ambig_part_size; ++i) {
309  if (!(token = strtok_r(nullptr, kAmbigDelimiters, &next_token))) break;
310  strcat(replacement_string, token);
311  if (!unicharset.contains_unichar(token)) {
312  if (debug_level) tprintf(kIllegalUnicharMsg, token);
313  break;
314  }
315  }
316  if (i != *replacement_ambig_part_size) {
317  if (debug_level) tprintf(kIllegalMsg, line_num);
318  return false;
319  }
320  if (version > 0) {
321  // The next field being true indicates that the abiguity should
322  // always be substituted (e.g. '' should always be changed to ").
323  // For such "certain" n -> m ambigs tesseract will insert character
324  // fragments for the n pieces in the unicharset. AmbigsFound()
325  // will then replace the incorrect ngram with the character
326  // fragments of the correct character (or ngram if m > 1).
327  // Note that if m > 1, an ngram will be inserted into the
328  // modified word, not the individual unigrams. Tesseract
329  // has limited support for ngram unichar (e.g. dawg permuter).
330  if (!(token = strtok_r(nullptr, kAmbigDelimiters, &next_token)) ||
331  !sscanf(token, "%d", type)) {
332  if (debug_level) tprintf(kIllegalMsg, line_num);
333  return false;
334  }
335  }
336  return true;
337 }
338 
339 bool UnicharAmbigs::InsertIntoTable(
340  UnicharAmbigsVector &table, int test_ambig_part_size,
341  UNICHAR_ID *test_unichar_ids, int replacement_ambig_part_size,
342  const char *replacement_string, int type,
343  AmbigSpec *ambig_spec, UNICHARSET *unicharset) {
344  ambig_spec->type = static_cast<AmbigType>(type);
345  if (test_ambig_part_size == 1 && replacement_ambig_part_size == 1 &&
346  unicharset->to_lower(test_unichar_ids[0]) ==
347  unicharset->to_lower(unicharset->unichar_to_id(replacement_string))) {
348  ambig_spec->type = CASE_AMBIG;
349  }
350 
351  ambig_spec->wrong_ngram_size =
352  UnicharIdArrayUtils::copy(test_unichar_ids, ambig_spec->wrong_ngram);
353 
354  // Since we need to maintain a constant number of unichar positions in
355  // order to construct ambig_blob_choices vector in NoDangerousAmbig(), for
356  // each n->m ambiguity we will have to place n character fragments of the
357  // correct ngram into the corresponding positions in the vector (e.g. given
358  // "vvvvw" and vvvv->ww we will place v and |ww|0|4 into position 0, v and
359  // |ww|1|4 into position 1 and so on. The correct ngram is reconstructed
360  // from fragments by dawg_permute_and_select().
361 
362  // Insert the corresponding correct ngram into the unicharset.
363  // Unicharset code assumes that the "base" ngram is inserted into
364  // the unicharset before fragments of this ngram are inserted.
365  unicharset->unichar_insert(replacement_string, OldUncleanUnichars::kTrue);
366  ambig_spec->correct_ngram_id =
367  unicharset->unichar_to_id(replacement_string);
368  if (replacement_ambig_part_size > 1) {
369  unicharset->set_isngram(ambig_spec->correct_ngram_id, true);
370  }
371  // Add the corresponding fragments of the wrong ngram to unicharset.
372  int i;
373  for (i = 0; i < test_ambig_part_size; ++i) {
374  UNICHAR_ID unichar_id;
375  if (test_ambig_part_size == 1) {
376  unichar_id = ambig_spec->correct_ngram_id;
377  } else {
378  STRING frag_str = CHAR_FRAGMENT::to_string(
379  replacement_string, i, test_ambig_part_size, false);
380  unicharset->unichar_insert(frag_str.c_str(), OldUncleanUnichars::kTrue);
381  unichar_id = unicharset->unichar_to_id(frag_str.c_str());
382  }
383  ambig_spec->correct_fragments[i] = unichar_id;
384  }
385  ambig_spec->correct_fragments[i] = INVALID_UNICHAR_ID;
386 
387  // Add AmbigSpec for this ambiguity to the corresponding AmbigSpec_LIST.
388  // Keep AmbigSpec_LISTs sorted by AmbigSpec.wrong_ngram.
389  if (table[test_unichar_ids[0]] == nullptr) {
390  table[test_unichar_ids[0]] = new AmbigSpec_LIST();
391  }
392  if (table[test_unichar_ids[0]]->add_sorted(
393  AmbigSpec::compare_ambig_specs, true, ambig_spec))
394  return true;
395  delete ambig_spec;
396  return false;
397 }
398 
399 } // namespace tesseract
tesseract::AmbigSpec::wrong_ngram
UNICHAR_ID wrong_ngram[MAX_AMBIG_SIZE+1]
Definition: ambigs.h:124
UNICHARSET::encode_string
bool encode_string(const char *str, bool give_up_on_failure, GenericVector< UNICHAR_ID > *encoding, GenericVector< char > *lengths, int *encoded_length) const
Definition: unicharset.cpp:258
tesseract::CASE_AMBIG
Definition: ambigs.h:42
tesseract::DEFINITE_AMBIG
Definition: ambigs.h:40
tesseract::File::Open
static FILE * Open(const std::string &filename, const std::string &mode)
Definition: fileio.cpp:54
ASSERT_HOST
#define ASSERT_HOST(x)
Definition: errcode.h:87
GenericVector::insert
void insert(const T &t, int index)
Definition: genericvector.h:750
OldUncleanUnichars::kTrue
tesseract::UnicharAmbigsVector
GenericVector< AmbigSpec_LIST * > UnicharAmbigsVector
Definition: ambigs.h:134
STRING
Definition: strngs.h:45
tesseract::UnicharAmbigs::LoadUnicharAmbigs
void LoadUnicharAmbigs(const UNICHARSET &encoder_set, TFile *ambigs_file, int debug_level, bool use_ambigs_for_adaption, UNICHARSET *unicharset)
Definition: ambigs.cpp:75
tesseract::kMaxAmbigStringSize
const int kMaxAmbigStringSize
Definition: ambigs.cpp:41
tesseract::TFile::Rewind
void Rewind()
Definition: serialis.cpp:304
MAX_AMBIG_SIZE
#define MAX_AMBIG_SIZE
Definition: ambigs.h:31
tesseract::NOT_AMBIG
Definition: ambigs.h:38
tesseract::UnicharAmbigs::LoadUniversal
void LoadUniversal(const UNICHARSET &encoder_set, UNICHARSET *unicharset)
Definition: ambigs.cpp:68
tesseract::AmbigSpec::correct_fragments
UNICHAR_ID correct_fragments[MAX_AMBIG_SIZE+1]
Definition: ambigs.h:125
tesseract::AmbigSpec::correct_ngram_id
UNICHAR_ID correct_ngram_id
Definition: ambigs.h:126
tesseract::AmbigSpec::AmbigSpec
AmbigSpec()
Definition: ambigs.cpp:43
tesseract::UnicharIdVector
GenericVector< UNICHAR_ID > UnicharIdVector
Definition: ambigs.h:35
GenericVector::push_back
int push_back(T object)
Definition: genericvector.h:799
UNICHARSET::to_lower
UNICHAR_ID to_lower(UNICHAR_ID unichar_id) const
Definition: unicharset.h:694
UNICHARSET::debug_str
STRING debug_str(UNICHAR_ID id) const
Definition: unicharset.cpp:342
STRING::c_str
const char * c_str() const
Definition: strngs.cpp:192
file
Definition: include_gunit.h:22
chomp_string
void chomp_string(char *str)
Definition: helpers.h:75
tesseract::kUniversalAmbigsFile
const char kUniversalAmbigsFile[]
Definition: universalambigs.cpp:24
UNICHARSET::unichar_to_id
UNICHAR_ID unichar_to_id(const char *const unichar_repr) const
Definition: unicharset.cpp:209
tesseract::TFile
Definition: serialis.h:75
tesseract::UnicharAmbigs
Definition: ambigs.h:136
UNICHARSET
Definition: unicharset.h:145
tesseract::AmbigSpec::wrong_ngram_size
int wrong_ngram_size
Definition: ambigs.h:128
helpers.h
tesseract
Definition: baseapi.h:65
CHAR_FRAGMENT::to_string
STRING to_string() const
Definition: unicharset.h:79
UNICHAR_ID
int UNICHAR_ID
Definition: unichar.h:36
GenericVector< UNICHAR_ID >
tesseract::UnicharIdArrayUtils::print
static void print(const UNICHAR_ID array[], const UNICHARSET &unicharset)
Definition: ambigs.h:91
tesseract::AmbigSpec
Definition: ambigs.h:107
UNICHAR_LEN
#define UNICHAR_LEN
Definition: unichar.h:32
tesseract::ksizeofUniversalAmbigsFile
const int ksizeofUniversalAmbigsFile
Definition: universalambigs.cpp:19036
UNICHARSET::set_isngram
void set_isngram(UNICHAR_ID unichar_id, bool value)
Definition: unicharset.h:446
tesseract::REPLACE_AMBIG
Definition: ambigs.h:39
universalambigs.h
UNICHARSET::contains_unichar
bool contains_unichar(const char *const unichar_repr) const
Definition: unicharset.cpp:670
tesseract::TFile::FGets
char * FGets(char *buffer, int buffer_size)
Definition: serialis.cpp:262
tesseract::AmbigSpec::compare_ambig_specs
static int compare_ambig_specs(const void *spec1, const void *spec2)
Definition: ambigs.h:115
tprintf
DLLSYM void tprintf(const char *format,...)
Definition: tprintf.cpp:34
tesstrain_utils.type
type
Definition: tesstrain_utils.py:141
ambigs.h
GenericVector::size
int size() const
Definition: genericvector.h:71
tesseract::AmbigSpec::type
AmbigType type
Definition: ambigs.h:127
UNICHARSET::unichar_insert
void unichar_insert(const char *const unichar_repr, OldUncleanUnichars old_style)
Definition: unicharset.cpp:625
ELISTIZE
#define ELISTIZE(CLASSNAME)
Definition: elst.h:919
tesseract::UnicharIdArrayUtils::copy
static int copy(const UNICHAR_ID src[], UNICHAR_ID dst[])
Definition: ambigs.h:81