tesseract  4.0.0-1-g2a2b
Public Member Functions | Static Public Member Functions | Static Public Attributes | List of all members
tesseract::UnicharCompress Class Reference

#include <unicharcompress.h>

Public Member Functions

 UnicharCompress ()
 
 UnicharCompress (const UnicharCompress &src)
 
 ~UnicharCompress ()
 
UnicharCompress & operator= (const UnicharCompress &src)
 
bool ComputeEncoding (const UNICHARSET &unicharset, int null_id, STRING *radical_stroke_table)
 
void SetupPassThrough (const UNICHARSET &unicharset)
 
void SetupDirect (const GenericVector< RecodedCharID > &codes)
 
int code_range () const
 
int EncodeUnichar (int unichar_id, RecodedCharID *code) const
 
int DecodeUnichar (const RecodedCharID &code) const
 
bool IsValidFirstCode (int code) const
 
const GenericVector< int > * GetNextCodes (const RecodedCharID &code) const
 
const GenericVector< int > * GetFinalCodes (const RecodedCharID &code) const
 
bool Serialize (TFile *fp) const
 
bool DeSerialize (TFile *fp)
 
STRING GetEncodingAsString (const UNICHARSET &unicharset) const
 

Static Public Member Functions

static bool DecomposeHangul (int unicode, int *leading, int *vowel, int *trailing)
 

Static Public Attributes

static const int kFirstHangul = 0xac00
 
static const int kNumHangul = 11172
 
static const int kLCount = 19
 
static const int kVCount = 21
 
static const int kTCount = 28
 

Detailed Description

Definition at line 128 of file unicharcompress.h.

Constructor & Destructor Documentation

◆ UnicharCompress() [1/2]

tesseract::UnicharCompress::UnicharCompress ( )

Definition at line 87 of file unicharcompress.cpp.

87 : code_range_(0) {}

◆ UnicharCompress() [2/2]

tesseract::UnicharCompress::UnicharCompress ( const UnicharCompress &  src)

Definition at line 88 of file unicharcompress.cpp.

88 { *this = src; }

◆ ~UnicharCompress()

tesseract::UnicharCompress::~UnicharCompress ( )

Definition at line 89 of file unicharcompress.cpp.

89 { Cleanup(); }

Member Function Documentation

◆ code_range()

int tesseract::UnicharCompress::code_range ( ) const
inline

Definition at line 161 of file unicharcompress.h.

161 { return code_range_; }

◆ ComputeEncoding()

bool tesseract::UnicharCompress::ComputeEncoding ( const UNICHARSET &  unicharset,
int  null_id,
STRING *  radical_stroke_table 
)

Definition at line 102 of file unicharcompress.cpp.

103  {
104  RSMap radical_map;
105  if (radical_stroke_table != nullptr &&
106  !DecodeRadicalTable(radical_stroke_table, &radical_map))
107  return false;
108  encoder_.clear();
109  UNICHARSET direct_set;
110  // To avoid unused codes, clear the special codes from the direct_set.
111  direct_set.clear();
112  // Always keep space as 0;
113  direct_set.unichar_insert(" ", OldUncleanUnichars::kTrue);
114  // Null char is next if we have one.
115  if (null_id >= 0) {
116  direct_set.unichar_insert(kNullChar);
117  }
118  RSCounts radical_counts;
119  // In the initial map, codes [0, unicharset.size()) are
120  // reserved for non-han/hangul sequences of 1 or more unicodes.
121  int hangul_offset = unicharset.size();
122  // Hangul takes the next range [hangul_offset, hangul_offset + kTotalJamos).
123  const int kTotalJamos = kLCount + kVCount + kTCount;
124  // Han takes the codes beyond hangul_offset + kTotalJamos. Since it is hard
125  // to measure the number of radicals and strokes, initially we use the same
126  // code range for all 3 Han code positions, and fix them after.
127  int han_offset = hangul_offset + kTotalJamos;
128  for (int u = 0; u <= unicharset.size(); ++u) {
129  // We special-case allow null_id to be equal to unicharset.size() in case
130  // there is no space in unicharset for it.
131  if (u == unicharset.size() && u != null_id) break; // Finished
132  RecodedCharID code;
133  // Convert to unicodes.
134  std::vector<char32> unicodes;
135  std::string cleaned;
136  if (u < unicharset.size())
137  cleaned = UNICHARSET::CleanupString(unicharset.id_to_unichar(u));
138  if (u < unicharset.size() &&
139  (unicodes = UNICHAR::UTF8ToUTF32(cleaned.c_str())).size() == 1) {
140  // Check single unicodes for Hangul/Han and encode if so.
141  int unicode = unicodes[0];
142  int leading, vowel, trailing;
143  auto it = radical_map.find(unicode);
144  if (it != radical_map.end()) {
145  // This is Han. Use the radical codes directly.
146  int num_radicals = it->second->size();
147  for (int c = 0; c < num_radicals; ++c) {
148  code.Set(c, han_offset + (*it->second)[c]);
149  }
150  int pre_hash = RadicalPreHash(*it->second);
151  int num_samples = radical_counts[pre_hash]++;
152  if (num_samples > 0)
153  code.Set(num_radicals, han_offset + num_samples + kRadicalRadix);
154  } else if (DecomposeHangul(unicode, &leading, &vowel, &trailing)) {
155  // This is Hangul. Since we know the exact size of each part at compile
156  // time, it gets the bottom set of codes.
157  code.Set3(leading + hangul_offset, vowel + kLCount + hangul_offset,
158  trailing + kLCount + kVCount + hangul_offset);
159  }
160  }
161  // If the code is still empty, it wasn't Han or Hangul.
162  if (code.length() == 0) {
163  // Special cases.
164  if (u == UNICHAR_SPACE) {
165  code.Set(0, 0); // Space.
166  } else if (u == null_id || (unicharset.has_special_codes() &&
167  u < SPECIAL_UNICHAR_CODES_COUNT)) {
168  code.Set(0, direct_set.unichar_to_id(kNullChar));
169  } else {
170  // Add the direct_set unichar-ids of the unicodes in sequence to the
171  // code.
172  for (int i = 0; i < unicodes.size(); ++i) {
173  int position = code.length();
174  if (position >= RecodedCharID::kMaxCodeLen) {
175  tprintf("Unichar %d=%s is too long to encode!!\n", u,
176  unicharset.id_to_unichar(u));
177  return false;
178  }
179  int uni = unicodes[i];
180  UNICHAR unichar(uni);
181  char* utf8 = unichar.utf8_str();
182  if (!direct_set.contains_unichar(utf8))
183  direct_set.unichar_insert(utf8);
184  code.Set(position, direct_set.unichar_to_id(utf8));
185  delete[] utf8;
186  if (direct_set.size() >
187  unicharset.size() + !unicharset.has_special_codes()) {
188  // Code space got bigger!
189  tprintf("Code space expanded from original unicharset!!\n");
190  return false;
191  }
192  }
193  }
194  }
195  encoder_.push_back(code);
196  }
197  // Now renumber Han to make all codes unique. We already added han_offset to
198  // all Han. Now separate out the radical, stroke, and count codes for Han.
199  int code_offset = 0;
200  for (int i = 0; i < RecodedCharID::kMaxCodeLen; ++i) {
201  int max_offset = 0;
202  for (int u = 0; u < unicharset.size(); ++u) {
203  RecodedCharID* code = &encoder_[u];
204  if (code->length() <= i) continue;
205  max_offset = std::max(max_offset, (*code)(i)-han_offset);
206  code->Set(i, (*code)(i) + code_offset);
207  }
208  if (max_offset == 0) break;
209  code_offset += max_offset + 1;
210  }
211  DefragmentCodeValues(null_id >= 0 ? 1 : -1);
212  SetupDecoder();
213  return true;
214 }
tesseract::RecodedCharID::kMaxCodeLen
static const int kMaxCodeLen
Definition: unicharcompress.h:37
UNICHARSET::unichar_to_id
UNICHAR_ID unichar_to_id(const char *const unichar_repr) const
Definition: unicharset.cpp:209
UNICHARSET::size
int size() const
Definition: unicharset.h:336
tesseract::kRadicalRadix
const int kRadicalRadix
Definition: unicharcompress.cpp:32
UNICHARSET::unichar_insert
void unichar_insert(const char *const unichar_repr, OldUncleanUnichars old_style)
Definition: unicharset.cpp:625
UNICHARSET::contains_unichar
bool contains_unichar(const char *const unichar_repr) const
Definition: unicharset.cpp:670
tprintf
DLLSYM void tprintf(const char *format,...)
Definition: tprintf.cpp:37
tesseract::kNullChar
const char * kNullChar
Definition: unicharcompress.cpp:30
tesseract::RSCounts
std::unordered_map< int, int > RSCounts
Definition: unicharcompress.cpp:49
UNICHARSET::CleanupString
static std::string CleanupString(const char *utf8_str)
Definition: unicharset.h:241
tesseract::UNICHAR::UTF8ToUTF32
static std::vector< char32 > UTF8ToUTF32(const char *utf8_str)
Definition: unichar.cpp:213
tesseract::UnicharCompress::DecomposeHangul
static bool DecomposeHangul(int unicode, int *leading, int *vowel, int *trailing)
Definition: unicharcompress.cpp:350
UNICHARSET::id_to_unichar
const char * id_to_unichar(UNICHAR_ID id) const
Definition: unicharset.cpp:290
UNICHARSET::clear
void clear()
Definition: unicharset.h:301
tesseract::RSMap
std::unordered_map< int, std::unique_ptr< std::vector< int > >> RSMap
Definition: unicharcompress.cpp:47
UNICHARSET::has_special_codes
bool has_special_codes() const
Definition: unicharset.h:717

◆ DecodeUnichar()

int tesseract::UnicharCompress::DecodeUnichar ( const RecodedCharID &  code) const

Definition at line 293 of file unicharcompress.cpp.

293  {
294  int len = code.length();
295  if (len <= 0 || len > RecodedCharID::kMaxCodeLen) return INVALID_UNICHAR_ID;
296  auto it = decoder_.find(code);
297  if (it == decoder_.end()) return INVALID_UNICHAR_ID;
298  return it->second;
299 }
tesseract::RecodedCharID::kMaxCodeLen
static const int kMaxCodeLen
Definition: unicharcompress.h:37

◆ DecomposeHangul()

bool tesseract::UnicharCompress::DecomposeHangul ( int  unicode,
int *  leading,
int *  vowel,
int *  trailing 
)
static

Definition at line 350 of file unicharcompress.cpp.

351  {
352  if (unicode < kFirstHangul) return false;
353  int offset = unicode - kFirstHangul;
354  if (offset >= kNumHangul) return false;
355  const int kNCount = kVCount * kTCount;
356  *leading = offset / kNCount;
357  *vowel = (offset % kNCount) / kTCount;
358  *trailing = offset % kTCount;
359  return true;
360 }
tesseract::UnicharCompress::kFirstHangul
static const int kFirstHangul
Definition: unicharcompress.h:136

◆ DeSerialize()

bool tesseract::UnicharCompress::DeSerialize ( TFile *  fp)

Definition at line 307 of file unicharcompress.cpp.

307  {
308  if (!encoder_.DeSerializeClasses(fp)) return false;
309  ComputeCodeRange();
310  SetupDecoder();
311  return true;
312 }

◆ EncodeUnichar()

int tesseract::UnicharCompress::EncodeUnichar ( int  unichar_id,
RecodedCharID *  code 
) const

Definition at line 285 of file unicharcompress.cpp.

285  {
286  if (unichar_id < 0 || unichar_id >= encoder_.size()) return 0;
287  *code = encoder_[unichar_id];
288  return code->length();
289 }

◆ GetEncodingAsString()

STRING tesseract::UnicharCompress::GetEncodingAsString ( const UNICHARSET &  unicharset) const

Definition at line 321 of file unicharcompress.cpp.

322  {
323  STRING encoding;
324  for (int c = 0; c < encoder_.size(); ++c) {
325  const RecodedCharID& code = encoder_[c];
326  if (0 < c && c < SPECIAL_UNICHAR_CODES_COUNT && code == encoder_[c - 1]) {
327  // Don't show the duplicate entry.
328  continue;
329  }
330  encoding.add_str_int("", code(0));
331  for (int i = 1; i < code.length(); ++i) {
332  encoding.add_str_int(",", code(i));
333  }
334  encoding += "\t";
335  if (c >= unicharset.size() || (0 < c && c < SPECIAL_UNICHAR_CODES_COUNT &&
336  unicharset.has_special_codes())) {
337  encoding += kNullChar;
338  } else {
339  encoding += unicharset.id_to_unichar(c);
340  }
341  encoding += "\n";
342  }
343  return encoding;
344 }
UNICHARSET::size
int size() const
Definition: unicharset.h:336
tesseract::kNullChar
const char * kNullChar
Definition: unicharcompress.cpp:30
STRING::add_str_int
void add_str_int(const char *str, int number)
Definition: strngs.cpp:379
STRING
Definition: strngs.h:45
UNICHARSET::id_to_unichar
const char * id_to_unichar(UNICHAR_ID id) const
Definition: unicharset.cpp:290
UNICHARSET::has_special_codes
bool has_special_codes() const
Definition: unicharset.h:717

◆ GetFinalCodes()

const GenericVector<int>* tesseract::UnicharCompress::GetFinalCodes ( const RecodedCharID &  code) const
inline

Definition at line 179 of file unicharcompress.h.

179  {
180  auto it = final_codes_.find(code);
181  return it == final_codes_.end() ? nullptr : it->second;
182  }

◆ GetNextCodes()

const GenericVector<int>* tesseract::UnicharCompress::GetNextCodes ( const RecodedCharID &  code) const
inline

Definition at line 173 of file unicharcompress.h.

173  {
174  auto it = next_codes_.find(code);
175  return it == next_codes_.end() ? nullptr : it->second;
176  }

◆ IsValidFirstCode()

bool tesseract::UnicharCompress::IsValidFirstCode ( int  code) const
inline

Definition at line 170 of file unicharcompress.h.

170 { return is_valid_start_[code]; }

◆ operator=()

UnicharCompress & tesseract::UnicharCompress::operator= ( const UnicharCompress &  src)

Definition at line 90 of file unicharcompress.cpp.

90  {
91  Cleanup();
92  encoder_ = src.encoder_;
93  code_range_ = src.code_range_;
94  SetupDecoder();
95  return *this;
96 }

◆ Serialize()

bool tesseract::UnicharCompress::Serialize ( TFile *  fp) const

Definition at line 302 of file unicharcompress.cpp.

302  {
303  return encoder_.SerializeClasses(fp);
304 }

◆ SetupDirect()

void tesseract::UnicharCompress::SetupDirect ( const GenericVector< RecodedCharID > &  codes)

Definition at line 235 of file unicharcompress.cpp.

235  {
236  encoder_ = codes;
237  ComputeCodeRange();
238  SetupDecoder();
239 }

◆ SetupPassThrough()

void tesseract::UnicharCompress::SetupPassThrough ( const UNICHARSET &  unicharset)

Definition at line 218 of file unicharcompress.cpp.

218  {
219  GenericVector<RecodedCharID> codes;
220  for (int u = 0; u < unicharset.size(); ++u) {
221  RecodedCharID code;
222  code.Set(0, u);
223  codes.push_back(code);
224  }
225  if (!unicharset.has_special_codes()) {
226  RecodedCharID code;
227  code.Set(0, unicharset.size());
228  codes.push_back(code);
229  }
230  SetupDirect(codes);
231 }
tesseract::UnicharCompress::SetupDirect
void SetupDirect(const GenericVector< RecodedCharID > &codes)
Definition: unicharcompress.cpp:235
UNICHARSET::size
int size() const
Definition: unicharset.h:336
GenericVector::push_back
int push_back(T object)
Definition: genericvector.h:799
UNICHARSET::has_special_codes
bool has_special_codes() const
Definition: unicharset.h:717

Member Data Documentation

◆ kFirstHangul

const int tesseract::UnicharCompress::kFirstHangul = 0xac00
static

Definition at line 136 of file unicharcompress.h.

◆ kLCount

const int tesseract::UnicharCompress::kLCount = 19
static

Definition at line 141 of file unicharcompress.h.

◆ kNumHangul

const int tesseract::UnicharCompress::kNumHangul = 11172
static

Definition at line 138 of file unicharcompress.h.

◆ kTCount

const int tesseract::UnicharCompress::kTCount = 28
static

Definition at line 143 of file unicharcompress.h.

◆ kVCount

const int tesseract::UnicharCompress::kVCount = 21
static

Definition at line 142 of file unicharcompress.h.

The documentation for this class was generated from the following files: