17 #include "allheaders.h"
39 TrainingSampleSet::FontClassInfo::FontClassInfo()
40 : num_raw_samples(0), canonical_sample(-1), canonical_dist(0.0f) {
45 if (fwrite(&num_raw_samples,
sizeof(num_raw_samples), 1, fp) != 1)
47 if (fwrite(&canonical_sample,
sizeof(canonical_sample), 1, fp) != 1)
49 if (fwrite(&canonical_dist,
sizeof(canonical_dist), 1, fp) != 1)
return false;
50 if (!samples.Serialize(fp))
return false;
56 if (fread(&num_raw_samples,
sizeof(num_raw_samples), 1, fp) != 1)
58 if (fread(&canonical_sample,
sizeof(canonical_sample), 1, fp) != 1)
60 if (fread(&canonical_dist,
sizeof(canonical_dist), 1, fp) != 1)
return false;
61 if (!samples.DeSerialize(swap, fp))
return false;
63 ReverseN(&num_raw_samples,
sizeof(num_raw_samples));
64 ReverseN(&canonical_sample,
sizeof(canonical_sample));
65 ReverseN(&canonical_dist,
sizeof(canonical_dist));
71 : num_raw_samples_(0), unicharset_size_(0),
72 font_class_array_(nullptr), fontinfo_table_(font_table) {
76 delete font_class_array_;
81 if (!samples_.Serialize(fp))
return false;
83 if (!font_id_map_.
Serialize(fp))
return false;
84 int8_t not_null = font_class_array_ !=
nullptr;
85 if (fwrite(¬_null,
sizeof(not_null), 1, fp) != 1)
return false;
95 if (!samples_.DeSerialize(swap, fp))
return false;
96 num_raw_samples_ = samples_.size();
98 if (!font_id_map_.
DeSerialize(swap, fp))
return false;
99 delete font_class_array_;
100 font_class_array_ =
nullptr;
102 if (fread(¬_null,
sizeof(not_null), 1, fp) != 1)
return false;
108 unicharset_size_ = unicharset_.
size();
115 tprintf(
"Failed to load unicharset from file %s\n"
116 "Building unicharset from scratch...\n",
123 unicharset_size_ = unicharset_.
size();
133 tprintf(
"Error: Size of unicharset in TrainingSampleSet::AddSample is "
134 "greater than MAX_NUM_CLASSES\n");
146 sample->set_class_id(unichar_id);
147 samples_.push_back(
sample);
148 num_raw_samples_ = samples_.size();
149 unicharset_size_ = unicharset_.
size();
157 bool randomize)
const {
159 if (font_id < 0 || class_id < 0 ||
160 font_id >= font_id_map_.
SparseSize() || class_id >= unicharset_size_) {
168 return (*font_class_array_)(font_index, class_id).samples.size();
175 return samples_[index];
184 if (font_index < 0)
return nullptr;
185 int sample_index = (*font_class_array_)(font_index, class_id).samples[index];
186 return samples_[sample_index];
195 if (font_index < 0)
return nullptr;
196 int sample_index = (*font_class_array_)(font_index, class_id).samples[index];
197 return samples_[sample_index];
205 sample.bounding_box(),
sample.page_num(), &boxfile_str);
206 return STRING(fontinfo_table_.
get(
sample.font_id()).name) +
" " + boxfile_str;
212 int font_id,
int class_id)
const {
215 return (*font_class_array_)(font_index, class_id).cloud_features;
220 int font_id,
int class_id)
const {
223 return (*font_class_array_)(font_index, class_id).canonical_features;
238 double dist_sum = 0.0;
240 const bool debug =
false;
243 for (
int i = 0; i < num_fonts1; ++i) {
245 for (
int j = 0; j < num_fonts2; ++j) {
255 for (
int i = 0; i < num_fonts1; ++i) {
257 for (
int j = 0; j < num_fonts2; ++j) {
261 tprintf(
"Cluster dist %d %d %d %d = %g\n",
273 int num_samples = std::max(num_fonts1, num_fonts2);
274 for (
int i = 0; i <
num_samples; ++i, index += increment) {
275 int f1 = uf1.
font_ids[i % num_fonts1];
276 int f2 = uf2.
font_ids[index % num_fonts2];
278 tprintf(
"Cluster dist %d %d %d %d = %g\n",
285 if (dist_count == 0) {
290 return dist_sum / dist_count;
297 int font_id2,
int class_id2,
302 if (font_index1 < 0 || font_index2 < 0)
304 FontClassInfo& fc_info = (*font_class_array_)(font_index1, class_id1);
305 if (font_id1 == font_id2) {
307 if (fc_info.unichar_distance_cache.size() == 0)
308 fc_info.unichar_distance_cache.init_to_size(unicharset_size_, -1.0f);
309 if (fc_info.unichar_distance_cache[class_id2] < 0) {
314 fc_info.unichar_distance_cache[class_id2] = result;
316 FontClassInfo& fc_info2 = (*font_class_array_)(font_index2, class_id2);
317 if (fc_info2.unichar_distance_cache.size() == 0)
318 fc_info2.unichar_distance_cache.init_to_size(unicharset_size_, -1.0f);
319 fc_info2.unichar_distance_cache[class_id1] = result;
321 return fc_info.unichar_distance_cache[class_id2];
322 }
else if (class_id1 == class_id2) {
324 if (fc_info.font_distance_cache.size() == 0)
325 fc_info.font_distance_cache.init_to_size(font_id_map_.
CompactSize(),
327 if (fc_info.font_distance_cache[font_index2] < 0) {
332 fc_info.font_distance_cache[font_index2] = result;
334 FontClassInfo& fc_info2 = (*font_class_array_)(font_index2, class_id2);
335 if (fc_info2.font_distance_cache.size() == 0)
336 fc_info2.font_distance_cache.init_to_size(font_id_map_.
CompactSize(),
338 fc_info2.font_distance_cache[font_index1] = result;
340 return fc_info.font_distance_cache[font_index2];
345 while (cache_index < fc_info.distance_cache.size() &&
346 (fc_info.distance_cache[cache_index].unichar_id != class_id2 ||
347 fc_info.distance_cache[cache_index].font_id != font_id2))
349 if (cache_index == fc_info.distance_cache.size()) {
354 FontClassDistance fc_dist = { class_id2, font_id2, result };
355 fc_info.distance_cache.push_back(fc_dist);
358 FontClassInfo& fc_info2 = (*font_class_array_)(font_index2, class_id2);
359 fc_dist.unichar_id = class_id1;
360 fc_dist.font_id = font_id1;
361 fc_info2.distance_cache.push_back(fc_dist);
363 return fc_info.distance_cache[cache_index].distance;
368 int font_id1,
int class_id1,
int font_id2,
int class_id2,
376 return static_cast<float>(dist) / denominator;
382 static void AddNearFeatures(
const IntFeatureMap& feature_map,
int f,
int levels,
384 int prev_num_features = 0;
386 int num_features = 1;
387 for (
int level = 0; level < levels; ++level) {
388 for (
int i = prev_num_features; i < num_features; ++i) {
389 int feature = (*good_features)[i];
390 for (
int dir = -kNumOffsetMaps; dir <= kNumOffsetMaps; ++dir) {
391 if (dir == 0)
continue;
398 prev_num_features = num_features;
399 num_features = good_features->
size();
414 int font_id2,
int class_id2,
416 bool thorough)
const {
419 if (sample2 ==
nullptr)
424 if (cloud1.
size() == 0)
425 return canonical2.
size();
428 for (
int f = 0; f < canonical2.
size(); ++f) {
429 const int feature = canonical2[f];
434 AddNearFeatures(feature_map, feature, 1, &good_features);
437 for (i = 0; i < good_features.
size(); ++i) {
438 int good_f = good_features[i];
439 if (cloud1[good_f]) {
443 if (i < good_features.
size())
456 if (font_index < 0)
return -1;
457 return (*font_class_array_)(font_index, class_id).samples[index];
463 int font_id,
int class_id)
const {
466 if (font_index < 0)
return nullptr;
467 const int sample_index = (*font_class_array_)(font_index,
468 class_id).canonical_sample;
469 return sample_index >= 0 ? samples_[sample_index] :
nullptr;
477 if (font_index < 0)
return 0.0f;
478 if ((*font_class_array_)(font_index, class_id).canonical_sample >= 0)
479 return (*font_class_array_)(font_index, class_id).canonical_dist;
486 for (
int s = 0; s < samples_.size(); ++s)
493 sample->set_sample_index(-1);
498 using namespace std::placeholders;
500 num_raw_samples_ = samples_.size();
515 int compact_font_size = font_id_map_.
CompactSize();
517 delete font_class_array_;
520 compact_font_size, unicharset_size_, empty);
521 for (
int s = 0; s < samples_.size(); ++s) {
522 int font_id = samples_[s]->font_id();
523 int class_id = samples_[s]->class_id();
524 if (font_id < 0 || font_id >= font_id_map_.
SparseSize()) {
525 tprintf(
"Font id = %d/%d, class id = %d/%d on sample %d\n",
526 font_id, font_id_map_.
SparseSize(), class_id, unicharset_size_,
530 ASSERT_HOST(class_id >= 0 && class_id < unicharset_size_);
532 (*font_class_array_)(font_index, class_id).samples.push_back(s);
536 for (
int f = 0; f < compact_font_size; ++f) {
537 for (
int c = 0; c < unicharset_size_; ++c)
539 (*font_class_array_)(f, c).samples.size();
543 num_raw_samples_ = samples_.size();
551 for (
int s = 0; s < samples_.size(); ++s) {
552 const int font_id = samples_[s]->font_id();
553 while (font_id >= font_counts.
size())
555 ++font_counts[font_id];
557 font_id_map_.
Init(font_counts.
size(),
false);
558 for (
int f = 0; f < font_counts.
size(); ++f) {
559 font_id_map_.
SetMap(f, font_counts[f] > 0);
561 font_id_map_.
Setup();
576 double global_worst_dist = 0.0;
579 for (
int font_index = 0; font_index < font_size; ++font_index) {
581 for (
int c = 0; c < unicharset_size_; ++c) {
582 int samples_found = 0;
583 FontClassInfo& fcinfo = (*font_class_array_)(font_index, c);
584 if (fcinfo.samples.size() == 0 ||
586 fcinfo.canonical_sample = -1;
587 fcinfo.canonical_dist = 0.0f;
588 if (debug)
tprintf(
"Skipping class %d\n", c);
593 double min_max_dist = 2.0;
596 double max_max_dist = 0.0;
599 fcinfo.canonical_sample = fcinfo.samples[0];
600 fcinfo.canonical_dist = 0.0f;
601 for (
int i = 0; i < fcinfo.samples.size(); ++i) {
602 int s1 = fcinfo.samples[i];
604 f_table.
Set(features1, features1.
size(),
true);
605 double max_dist = 0.0;
610 for (
int j = 0; j < fcinfo.samples.size(); ++j) {
611 int s2 = fcinfo.samples[j];
612 if (samples_[s2]->class_id() != c ||
613 samples_[s2]->font_id() != font_id ||
618 if (dist > max_dist) {
620 if (dist > max_max_dist) {
629 f_table.
Set(features1, features1.
size(),
false);
630 samples_[s1]->set_max_dist(max_dist);
632 if (max_dist < min_max_dist) {
633 fcinfo.canonical_sample = s1;
634 fcinfo.canonical_dist = max_dist;
636 UpdateRange(max_dist, &min_max_dist, &max_max_dist);
638 if (max_max_dist > global_worst_dist) {
640 global_worst_dist = max_max_dist;
645 tprintf(
"Found %d samples of class %d=%s, font %d, "
646 "dist range [%g, %g], worst pair= %s, %s\n",
648 font_index, min_max_dist, max_max_dist,
655 tprintf(
"Global worst dist = %g, between sample %d and %d\n",
656 global_worst_dist, worst_s1, worst_s2);
668 for (
int font_index = 0; font_index < font_size; ++font_index) {
669 for (
int c = 0; c < unicharset_size_; ++c) {
670 FontClassInfo& fcinfo = (*font_class_array_)(font_index, c);
671 int sample_count = fcinfo.samples.size();
672 int min_samples = 2 * std::max(kSampleRandomSize, sample_count);
673 if (sample_count > 0 && sample_count < min_samples) {
674 int base_count = sample_count;
675 for (
int base_index = 0; sample_count < min_samples; ++sample_count) {
676 int src_index = fcinfo.samples[base_index++];
677 if (base_index >= base_count) base_index = 0;
679 sample_count % kSampleRandomSize);
680 int sample_index = samples_.size();
681 sample->set_sample_index(sample_index);
682 samples_.push_back(
sample);
683 fcinfo.samples.push_back(sample_index);
697 for (
int font_index = 0; font_index < font_size; ++font_index) {
699 for (
int c = 0; c < unicharset_size_; ++c) {
704 FontClassInfo& fcinfo = (*font_class_array_)(font_index, c);
705 fcinfo.canonical_features =
sample->indexed_features();
715 for (
int font_index = 0; font_index < font_size; ++font_index) {
717 for (
int c = 0; c < unicharset_size_; ++c) {
721 FontClassInfo& fcinfo = (*font_class_array_)(font_index, c);
722 fcinfo.cloud_features.Init(feature_space_size);
726 for (
int i = 0; i < sample_features.
size(); ++i)
727 fcinfo.cloud_features.SetBit(sample_features[i]);
735 for (
int f = 0; f < font_id_map_.
CompactSize(); ++f) {
754 for (
int f = 0; f < indexed_features.
size(); ++f) {
755 if (indexed_features[f] == f_index) {
756 sample->DisplayFeatures(color, window);