tesseract  5.0.0-alpha-619-ge9db
boxread.cpp
Go to the documentation of this file.
1 /**********************************************************************
2  * File: boxread.cpp
3  * Description: Read data from a box file.
4  * Author: Ray Smith
5  *
6  * (C) Copyright 2007, Google Inc.
7  ** Licensed under the Apache License, Version 2.0 (the "License");
8  ** you may not use this file except in compliance with the License.
9  ** You may obtain a copy of the License at
10  ** http://www.apache.org/licenses/LICENSE-2.0
11  ** Unless required by applicable law or agreed to in writing, software
12  ** distributed under the License is distributed on an "AS IS" BASIS,
13  ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  ** See the License for the specific language governing permissions and
15  ** limitations under the License.
16  *
17  **********************************************************************/
18 
19 #include "boxread.h"
20 #include <cstring> // for strchr, strcmp
21 #include <locale> // for std::locale::classic
22 #include <sstream> // for std::stringstream
23 #include <string> // for std::string
24 #include "errcode.h" // for ERRCODE, TESSEXIT
25 #include "fileerr.h" // for CANTOPENFILE
26 #include <tesseract/genericvector.h> // for GenericVector
27 #include <tesseract/helpers.h> // for chomp_string
28 #include "rect.h" // for TBOX
29 #include <tesseract/strngs.h> // for STRING
30 #include "tprintf.h" // for tprintf
31 #include <tesseract/unichar.h> // for UNICHAR
32 
33 // Special char code used to identify multi-blob labels.
34 static const char* kMultiBlobLabelCode = "WordStr";
35 
36 // Returns the box file name corresponding to the given image_filename.
37 static std::string BoxFileName(const char* image_filename) {
38  std::string box_filename = image_filename;
39  size_t length = box_filename.length();
40  std::string last = (length > 8) ? box_filename.substr(length - 8) : "";
41  if (last == ".bin.png" || last == ".nrm.png") {
42  box_filename.resize(length - 8);
43  } else {
44  size_t lastdot = box_filename.find_last_of('.');
45  if (lastdot < length) {
46  box_filename.resize(lastdot);
47  }
48  }
49  box_filename += ".box";
50  return box_filename;
51 }
52 
53 // Open the boxfile based on the given image filename.
54 FILE* OpenBoxFile(const STRING& fname) {
55  std::string filename = BoxFileName(fname.c_str());
56  FILE* box_file = nullptr;
57  if (!(box_file = fopen(filename.c_str(), "rb"))) {
58  CANTOPENFILE.error("read_next_box", TESSEXIT, "Can't open box file %s",
59  filename.c_str());
60  }
61  return box_file;
62 }
63 
64 // Reads all boxes from the given filename.
65 // Reads a specific target_page number if >= 0, or all pages otherwise.
66 // Skips blanks if skip_blanks is true.
67 // The UTF-8 label of the box is put in texts, and the full box definition as
68 // a string is put in box_texts, with the corresponding page number in pages.
69 // Each of the output vectors is optional (may be nullptr).
70 // Returns false if no boxes are found.
71 bool ReadAllBoxes(int target_page, bool skip_blanks, const STRING& filename,
72  GenericVector<TBOX>* boxes,
73  GenericVector<STRING>* texts,
74  GenericVector<STRING>* box_texts,
75  GenericVector<int>* pages) {
76  GenericVector<char> box_data;
77  if (!tesseract::LoadDataFromFile(BoxFileName(filename.c_str()).c_str(), &box_data))
78  return false;
79  // Convert the array of bytes to a string, so it can be used by the parser.
80  box_data.push_back('\0');
81  return ReadMemBoxes(target_page, skip_blanks, &box_data[0],
82  /*continue_on_failure*/ true, boxes, texts, box_texts,
83  pages);
84 }
85 
86 // Reads all boxes from the string. Otherwise, as ReadAllBoxes.
87 bool ReadMemBoxes(int target_page, bool skip_blanks, const char* box_data,
88  bool continue_on_failure,
89  GenericVector<TBOX>* boxes,
90  GenericVector<STRING>* texts,
91  GenericVector<STRING>* box_texts,
92  GenericVector<int>* pages) {
93  STRING box_str(box_data);
95  box_str.split('\n', &lines);
96  if (lines.empty()) return false;
97  int num_boxes = 0;
98  for (int i = 0; i < lines.size(); ++i) {
99  int page = 0;
100  STRING utf8_str;
101  TBOX box;
102  if (!ParseBoxFileStr(lines[i].c_str(), &page, &utf8_str, &box)) {
103  if (continue_on_failure)
104  continue;
105  else
106  return false;
107  }
108  if (skip_blanks && (utf8_str == " " || utf8_str == "\t")) continue;
109  if (target_page >= 0 && page != target_page) continue;
110  if (boxes != nullptr) boxes->push_back(box);
111  if (texts != nullptr) texts->push_back(utf8_str);
112  if (box_texts != nullptr) {
113  STRING full_text;
114  MakeBoxFileStr(utf8_str.c_str(), box, target_page, &full_text);
115  box_texts->push_back(full_text);
116  }
117  if (pages != nullptr) pages->push_back(page);
118  ++num_boxes;
119  }
120  return num_boxes > 0;
121 }
122 
123 // TODO(rays) convert all uses of ReadNextBox to use the new ReadAllBoxes.
124 // Box files are used ONLY DURING TRAINING, but by both processes of
125 // creating tr files with tesseract, and unicharset_extractor.
126 // ReadNextBox factors out the code to interpret a line of a box
127 // file so that applybox and unicharset_extractor interpret the same way.
128 // This function returns the next valid box file utf8 string and coords
129 // and returns true, or false on eof (and closes the file).
130 // It ignores the utf8 file signature ByteOrderMark (U+FEFF=EF BB BF), checks
131 // for valid utf-8 and allows space or tab between fields.
132 // utf8_str is set with the unichar string, and bounding box with the box.
133 // If there are page numbers in the file, it reads them all.
134 bool ReadNextBox(int *line_number, FILE* box_file,
135  STRING* utf8_str, TBOX* bounding_box) {
136  return ReadNextBox(-1, line_number, box_file, utf8_str, bounding_box);
137 }
138 
139 // As ReadNextBox above, but get a specific page number. (0-based)
140 // Use -1 to read any page number. Files without page number all
141 // read as if they are page 0.
142 bool ReadNextBox(int target_page, int *line_number, FILE* box_file,
143  STRING* utf8_str, TBOX* bounding_box) {
144  int page = 0;
145  char buff[kBoxReadBufSize]; // boxfile read buffer
146  char *buffptr = buff;
147 
148  while (fgets(buff, sizeof(buff) - 1, box_file)) {
149  (*line_number)++;
150 
151  buffptr = buff;
152  const auto *ubuf = reinterpret_cast<const unsigned char*>(buffptr);
153  if (ubuf[0] == 0xef && ubuf[1] == 0xbb && ubuf[2] == 0xbf)
154  buffptr += 3; // Skip unicode file designation.
155  // Check for blank lines in box file
156  if (*buffptr == '\n' || *buffptr == '\0') continue;
157  // Skip blank boxes.
158  if (*buffptr == ' ' || *buffptr == '\t') continue;
159  if (*buffptr != '\0') {
160  if (!ParseBoxFileStr(buffptr, &page, utf8_str, bounding_box)) {
161  tprintf("Box file format error on line %i; ignored\n", *line_number);
162  continue;
163  }
164  if (target_page >= 0 && target_page != page)
165  continue; // Not on the appropriate page.
166  return true; // Successfully read a box.
167  }
168  }
169  fclose(box_file);
170  return false; // EOF
171 }
172 
173 // Parses the given box file string into a page_number, utf8_str, and
174 // bounding_box. Returns true on a successful parse.
175 // The box file is assumed to contain box definitions, one per line, of the
176 // following format for blob-level boxes:
177 // <UTF8 str> <left> <bottom> <right> <top> <page id>
178 // and for word/line-level boxes:
179 // WordStr <left> <bottom> <right> <top> <page id> #<space-delimited word str>
180 // See applyybox.cpp for more information.
181 bool ParseBoxFileStr(const char* boxfile_str, int* page_number,
182  STRING* utf8_str, TBOX* bounding_box) {
183  *bounding_box = TBOX(); // Initialize it to empty.
184  *utf8_str = "";
185  char uch[kBoxReadBufSize];
186  const char *buffptr = boxfile_str;
187  // Read the unichar without messing up on Tibetan.
188  // According to issue 253 the utf-8 surrogates 85 and A0 are treated
189  // as whitespace by sscanf, so it is more reliable to just find
190  // ascii space and tab.
191  int uch_len = 0;
192  // Skip unicode file designation, if present.
193  const auto *ubuf = reinterpret_cast<const unsigned char*>(buffptr);
194  if (ubuf[0] == 0xef && ubuf[1] == 0xbb && ubuf[2] == 0xbf)
195  buffptr += 3;
196  // Allow a single blank as the UTF-8 string. Check for empty string and
197  // then blindly eat the first character.
198  if (*buffptr == '\0') return false;
199  do {
200  uch[uch_len++] = *buffptr++;
201  } while (*buffptr != '\0' && *buffptr != ' ' && *buffptr != '\t' &&
202  uch_len < kBoxReadBufSize - 1);
203  uch[uch_len] = '\0';
204  if (*buffptr != '\0') ++buffptr;
205  int x_min = INT_MAX;
206  int y_min = INT_MAX;
207  int x_max = INT_MIN;
208  int y_max = INT_MIN;
209  *page_number = 0;
210  std::stringstream stream(buffptr);
211  stream.imbue(std::locale::classic());
212  stream >> x_min;
213  stream >> y_min;
214  stream >> x_max;
215  stream >> y_max;
216  stream >> *page_number;
217  if (x_max < x_min || y_max < y_min) {
218  tprintf("Bad box coordinates in boxfile string! %s\n", ubuf);
219  return false;
220  }
221  // Test for long space-delimited string label.
222  if (strcmp(uch, kMultiBlobLabelCode) == 0 &&
223  (buffptr = strchr(buffptr, '#')) != nullptr) {
224  strncpy(uch, buffptr + 1, kBoxReadBufSize - 1);
225  uch[kBoxReadBufSize - 1] = '\0'; // Prevent buffer overrun.
226  chomp_string(uch);
227  uch_len = strlen(uch);
228  }
229  // Validate UTF8 by making unichars with it.
230  int used = 0;
231  while (used < uch_len) {
232  tesseract::UNICHAR ch(uch + used, uch_len - used);
233  int new_used = ch.utf8_len();
234  if (new_used == 0) {
235  tprintf("Bad UTF-8 str %s starts with 0x%02x at col %d\n",
236  uch + used, uch[used], used + 1);
237  return false;
238  }
239  used += new_used;
240  }
241  *utf8_str = uch;
242  if (x_min > x_max) Swap(&x_min, &x_max);
243  if (y_min > y_max) Swap(&y_min, &y_max);
244  bounding_box->set_to_given_coords(x_min, y_min, x_max, y_max);
245  return true; // Successfully read a box.
246 }
247 
248 // Creates a box file string from a unichar string, TBOX and page number.
249 void MakeBoxFileStr(const char* unichar_str, const TBOX& box, int page_num,
250  STRING* box_str) {
251  *box_str = unichar_str;
252  box_str->add_str_int(" ", box.left());
253  box_str->add_str_int(" ", box.bottom());
254  box_str->add_str_int(" ", box.right());
255  box_str->add_str_int(" ", box.top());
256  box_str->add_str_int(" ", page_num);
257 }
TBOX
Definition: cleanapi_test.cc:19
tesseract::UNICHAR::utf8_len
int utf8_len() const
Definition: unichar.h:79
string
std::string string
Definition: equationdetect_test.cc:21
strngs.h
TESSEXIT
Definition: errcode.h:42
STRING::add_str_int
void add_str_int(const char *str, int number)
Definition: strngs.cpp:370
boxread.h
OpenBoxFile
FILE * OpenBoxFile(const STRING &fname)
Definition: boxread.cpp:54
tesseract::LoadDataFromFile
bool LoadDataFromFile(const char *filename, GenericVector< char > *data)
Definition: genericvector.h:341
MakeBoxFileStr
void MakeBoxFileStr(const char *unichar_str, const TBOX &box, int page_num, STRING *box_str)
Definition: boxread.cpp:249
TBOX::top
int16_t top() const
Definition: rect.h:57
STRING
Definition: strngs.h:45
rect.h
TBOX::set_to_given_coords
void set_to_given_coords(int x_min, int y_min, int x_max, int y_max)
Definition: rect.h:270
tesseract::UNICHAR
Definition: unichar.h:59
ReadNextBox
bool ReadNextBox(int *line_number, FILE *box_file, STRING *utf8_str, TBOX *bounding_box)
Definition: boxread.cpp:134
genericvector.h
GenericVector::push_back
int push_back(T object)
Definition: genericvector.h:799
ParseBoxFileStr
bool ParseBoxFileStr(const char *boxfile_str, int *page_number, STRING *utf8_str, TBOX *bounding_box)
Definition: boxread.cpp:181
ERRCODE::error
void error(const char *caller, TessErrorLogCode action, const char *format,...) const
Definition: errcode.cpp:33
last
LIST last(LIST var_list)
Definition: oldlist.cpp:151
STRING::c_str
const char * c_str() const
Definition: strngs.cpp:192
fileerr.h
chomp_string
void chomp_string(char *str)
Definition: helpers.h:75
GenericVector::empty
bool empty() const
Definition: genericvector.h:86
TBOX::bottom
int16_t bottom() const
Definition: rect.h:64
ReadMemBoxes
bool ReadMemBoxes(int target_page, bool skip_blanks, const char *box_data, bool continue_on_failure, GenericVector< TBOX > *boxes, GenericVector< STRING > *texts, GenericVector< STRING > *box_texts, GenericVector< int > *pages)
Definition: boxread.cpp:87
helpers.h
tprintf.h
ReadAllBoxes
bool ReadAllBoxes(int target_page, bool skip_blanks, const STRING &filename, GenericVector< TBOX > *boxes, GenericVector< STRING > *texts, GenericVector< STRING > *box_texts, GenericVector< int > *pages)
Definition: boxread.cpp:71
GenericVector< TBOX >
kBoxReadBufSize
const int kBoxReadBufSize
Definition: boxread.h:30
TBOX::left
int16_t left() const
Definition: rect.h:71
unichar.h
CANTOPENFILE
constexpr ERRCODE CANTOPENFILE("Can't open file")
TBOX::right
int16_t right() const
Definition: rect.h:78
tprintf
DLLSYM void tprintf(const char *format,...)
Definition: tprintf.cpp:34
errcode.h
Swap
void Swap(T *p1, T *p2)
Definition: helpers.h:93
GenericVector::size
int size() const
Definition: genericvector.h:71
STRING::split
void split(char c, GenericVector< STRING > *splited)
Definition: strngs.cpp:275
TBOX
Definition: rect.h:33