tesseract  5.0.0-alpha-619-ge9db
unicharset_extractor.cpp
Go to the documentation of this file.
1 // File: unicharset_extractor.cpp
3 // Description: Unicode character/ligature set extractor.
4 // Author: Thomas Kielbus
5 //
6 // (C) Copyright 2006, Google Inc.
7 // Licensed under the Apache License, Version 2.0 (the "License");
8 // you may not use this file except in compliance with the License.
9 // You may obtain a copy of the License at
10 // http://www.apache.org/licenses/LICENSE-2.0
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
16 //
18 
19 // Given a list of box files or text files on the command line, this program
20 // normalizes the text according to command-line options and generates
21 // a unicharset.
22 
23 #include <cstdlib>
24 #include "boxread.h"
25 #include "commandlineflags.h"
26 #include "commontraining.h" // CheckSharedLibraryVersion
28 #include "lang_model_helpers.h"
29 #include "normstrngs.h"
30 #include <tesseract/strngs.h>
31 #include "unicharset.h"
33 
34 static STRING_PARAM_FLAG(output_unicharset, "unicharset", "Output file path");
35 static INT_PARAM_FLAG(norm_mode, 1,
36  "Normalization mode: 1=Combine graphemes, "
37  "2=Split graphemes, 3=Pure unicode");
38 
39 namespace tesseract {
40 
41 // Helper normalizes and segments the given strings according to norm_mode, and
42 // adds the segmented parts to unicharset.
43 static void AddStringsToUnicharset(const GenericVector<STRING>& strings,
44  int norm_mode, UNICHARSET* unicharset) {
45  for (int i = 0; i < strings.size(); ++i) {
46  std::vector<std::string> normalized;
48  static_cast<GraphemeNormMode>(norm_mode),
49  /*report_errors*/ true,
50  strings[i].c_str(), &normalized)) {
51  for (const std::string& normed : normalized) {
52 
53  // normed is a UTF-8 encoded string
54  if (normed.empty() || IsUTF8Whitespace(normed.c_str())) continue;
55  unicharset->unichar_insert(normed.c_str());
56  }
57  } else {
58  tprintf("Normalization failed for string '%s'\n", strings[i].c_str());
59  }
60  }
61 }
62 
63 static int Main(int argc, char** argv) {
64  UNICHARSET unicharset;
65  // Load input files
66  for (int arg = 1; arg < argc; ++arg) {
67  STRING file_data = tesseract::ReadFile(argv[arg], /*reader*/ nullptr);
68  if (file_data.length() == 0) continue;
70  if (ReadMemBoxes(-1, /*skip_blanks*/ true, &file_data[0],
71  /*continue_on_failure*/ false, /*boxes*/ nullptr,
72  &texts, /*box_texts*/ nullptr, /*pages*/ nullptr)) {
73  tprintf("Extracting unicharset from box file %s\n", argv[arg]);
74  } else {
75  tprintf("Extracting unicharset from plain text file %s\n", argv[arg]);
76  texts.truncate(0);
77  file_data.split('\n', &texts);
78  }
79  AddStringsToUnicharset(texts, FLAGS_norm_mode, &unicharset);
80  }
81  SetupBasicProperties(/*report_errors*/ true, /*decompose*/ false,
82  &unicharset);
83  // Write unicharset file.
84  if (unicharset.save_to_file(FLAGS_output_unicharset.c_str())) {
85  tprintf("Wrote unicharset file %s\n", FLAGS_output_unicharset.c_str());
86  } else {
87  tprintf("Cannot save unicharset file %s\n",
88  FLAGS_output_unicharset.c_str());
89  return EXIT_FAILURE;
90  }
91  return EXIT_SUCCESS;
92 }
93 
94 } // namespace tesseract
95 
96 int main(int argc, char** argv) {
97  tesseract::CheckSharedLibraryVersion();
98  if (argc > 1) {
99  tesseract::ParseCommandLineFlags(argv[0], &argc, &argv, true);
100  }
101  if (argc < 2) {
102  tprintf(
103  "Usage: %s [--output_unicharset filename] [--norm_mode mode]"
104  " box_or_text_file [...]\n",
105  argv[0]);
106  tprintf("Where mode means:\n");
107  tprintf(" 1=combine graphemes (use for Latin and other simple scripts)\n");
108  tprintf(" 2=split graphemes (use for Indic/Khmer/Myanmar)\n");
109  tprintf(" 3=pure unicode (use for Arabic/Hebrew/Thai/Tibetan)\n");
110  tprintf("Reads box or plain text files to extract the unicharset.\n");
111  return EXIT_FAILURE;
112  }
113  return tesseract::Main(argc, argv);
114 }
tesseract::OCRNorm::kNone
string
std::string string
Definition: equationdetect_test.cc:21
strngs.h
unicharset_training_utils.h
INT_PARAM_FLAG
#define INT_PARAM_FLAG(name, val, comment)
Definition: commandlineflags.h:25
commontraining.h
boxread.h
STRING_PARAM_FLAG
#define STRING_PARAM_FLAG(name, val, comment)
Definition: commandlineflags.h:37
tesseract::NormalizeCleanAndSegmentUTF8
bool NormalizeCleanAndSegmentUTF8(UnicodeNormMode u_mode, OCRNorm ocr_normalize, GraphemeNormMode g_mode, bool report_errors, const char *str8, std::vector< std::string > *graphemes)
Definition: normstrngs.cpp:188
STRING
Definition: strngs.h:45
tesseract::SetupBasicProperties
void SetupBasicProperties(bool report_errors, bool decompose, UNICHARSET *unicharset)
Definition: unicharset_training_utils.cpp:40
tesseract::ParseCommandLineFlags
void ParseCommandLineFlags(const char *usage, int *argc, char ***argv, const bool remove_flags)
Definition: commandlineflags.cpp:166
UNICHARSET::save_to_file
bool save_to_file(const char *const filename) const
Definition: unicharset.h:350
lang_model_helpers.h
genericvector.h
unicharset.h
UNICHARSET
Definition: unicharset.h:145
ReadMemBoxes
bool ReadMemBoxes(int target_page, bool skip_blanks, const char *box_data, bool continue_on_failure, GenericVector< TBOX > *boxes, GenericVector< STRING > *texts, GenericVector< STRING > *box_texts, GenericVector< int > *pages)
Definition: boxread.cpp:87
tesseract
Definition: baseapi.h:65
GenericVector< STRING >
normstrngs.h
tesseract::IsUTF8Whitespace
bool IsUTF8Whitespace(const char *text)
Definition: normstrngs.cpp:245
tesseract::ReadFile
STRING ReadFile(const std::string &filename, FileReader reader)
Definition: lang_model_helpers.cpp:57
STRING::length
int32_t length() const
Definition: strngs.cpp:187
GenericVector::truncate
void truncate(int size)
Definition: genericvector.h:132
tesseract::UnicodeNormMode::kNFC
tprintf
DLLSYM void tprintf(const char *format,...)
Definition: tprintf.cpp:34
commandlineflags.h
main
int main(int argc, char **argv)
Definition: unicharset_extractor.cpp:96
GenericVector::size
int size() const
Definition: genericvector.h:71
UNICHARSET::unichar_insert
void unichar_insert(const char *const unichar_repr, OldUncleanUnichars old_style)
Definition: unicharset.cpp:625
STRING::split
void split(char c, GenericVector< STRING > *splited)
Definition: strngs.cpp:275