tesseract::Validator Class Reference

#include <validator.h>

Inheritance diagram for tesseract::Validator:

Public Member Functions
virtual	~Validator ()

Static Public Member Functions
static bool	ValidateCleanAndSegment (GraphemeNormMode g_mode, bool report_errors, const std::vector< char32 > &src, std::vector< std::vector< char32 >> *dest)
static bool	IsZeroWidthMark (char32 ch)

Static Public Attributes
static const char32	kZeroWidthSpace = 0x200B
static const char32	kZeroWidthNonJoiner = 0x200C
static const char32	kZeroWidthJoiner = 0x200D
static const char32	kLeftToRightMark = 0x200E
static const char32	kRightToLeftMark = 0x200F
static const char32	kInvalid = 0xfffd

Protected Types
enum	CharClass {   CharClass::kConsonant = 'C', CharClass::kVowel = 'V', CharClass::kVirama = 'H', CharClass::kMatra = 'M',   CharClass::kMatraPiece = 'P', CharClass::kVowelModifier = 'D', CharClass::kZeroWidthNonJoiner = 'z', CharClass::kZeroWidthJoiner = 'Z',   CharClass::kVedicMark = 'v', CharClass::kNukta = 'N', CharClass::kRobat = 'R', CharClass::kOther = 'O',   CharClass::kWhitespace = ' ', CharClass::kCombiner = 'c' }
using	IndicPair = std::pair< CharClass, char32 >

Protected Member Functions
	Validator (ViramaScript script, bool report_errors)
bool	ValidateCleanAndSegmentInternal (GraphemeNormMode g_mode, const std::vector< char32 > &src, std::vector< std::vector< char32 >> *dest)
void	MoveResultsToDest (GraphemeNormMode g_mode, std::vector< std::vector< char32 >> *dest)
bool	IsSubscriptScript () const
bool	CodeOnlyToOutput ()
void	MultiCodePart (unsigned length)
bool	UseMultiCode (unsigned length)
virtual bool	ConsumeGraphemeIfValid ()=0
void	ComputeClassCodes (const std::vector< char32 > &text)
virtual CharClass	UnicodeToCharClass (char32 ch) const =0
void	Clear ()

Static Protected Member Functions
static std::unique_ptr< Validator >	ScriptValidator (ViramaScript script, bool report_errors)
static ViramaScript	MostFrequentViramaScript (const std::vector< char32 > &utf32)
static bool	IsVirama (char32 unicode)
static bool	IsVedicAccent (char32 unicode)

Protected Attributes
ViramaScript	script_
std::vector< IndicPair >	codes_
std::vector< std::vector< char32 > >	parts_
std::vector< char32 >	output_
unsigned	codes_used_
unsigned	output_used_
bool	report_errors_

Static Protected Attributes
static const int	kIndicCodePageSize = 128
static const char32	kMinIndicUnicode = 0x900
static const char32	kMaxSinhalaUnicode = 0xdff
static const char32	kMaxViramaScriptUnicode = 0x17ff
static const char32	kSinhalaVirama = 0xdca
static const char32	kMyanmarVirama = 0x1039
static const char32	kKhmerVirama = 0x17d2
static const char32	kJavaneseVirama = 0xa9c0
static const char32	kMaxJavaneseUnicode = 0xa9df

Detailed Description

Definition at line 86 of file validator.h.

Member Typedef Documentation

IndicPair

using tesseract::Validator::IndicPair = std::pair<CharClass, char32>

protected

Definition at line 148 of file validator.h.

Member Enumeration Documentation

CharClass

enum tesseract::Validator::CharClass

strongprotected

Enumerator
kConsonant
kVowel
kVirama
kMatra
kMatraPiece
kVowelModifier
kZeroWidthNonJoiner
kZeroWidthJoiner
kVedicMark
kNukta
kRobat
kOther
kWhitespace
kCombiner

Definition at line 126 of file validator.h.

      : script_(script),
        codes_used_(0),
        output_used_(0),
        report_errors_(report_errors) {}
 
  // Factory method that understands how to map script to the right subclass.
  static std::unique_ptr<Validator> ScriptValidator(ViramaScript script,
                                                    bool report_errors);
 
  // Internal version of the public static ValidateCleanAndSegment.
  // Validates and cleans the src vector of unicodes to the *dest, according to
  // its type and the given g_mode.

Constructor & Destructor Documentation

~Validator()

tesseract::Validator::~Validator ( )

virtualdefault

Validator()

tesseract::Validator::Validator ( ViramaScript  script, bool  report_errors  )

inlineprotected

Definition at line 150 of file validator.h.

Member Function Documentation

Clear()

void tesseract::Validator::Clear ( )

protected

Definition at line 214 of file validator.cpp.

                      {
  codes_.clear();
  parts_.clear();
  output_.clear();
  codes_used_ = 0;
  output_used_ = 0;
}

CodeOnlyToOutput()

bool tesseract::Validator::CodeOnlyToOutput ( )

inlineprotected

Definition at line 186 of file validator.h.

                                                   {output_[output_used_]});
    while (++output_used_ < output_.size()) {
      parts_.back().push_back(output_[output_used_]);
    }

ComputeClassCodes()

void tesseract::Validator::ComputeClassCodes ( const std::vector< char32 > &  text )

protected

Definition at line 206 of file validator.cpp.

                                                               {
  codes_.reserve(text.size());
  for (char32 c : text) {
    codes_.push_back(std::make_pair(UnicodeToCharClass(c), c));
  }
}

ConsumeGraphemeIfValid()

virtual bool tesseract::Validator::ConsumeGraphemeIfValid ( )

protectedpure virtual

Implemented in tesseract::ValidateJavanese, tesseract::ValidateIndic, tesseract::ValidateGrapheme, tesseract::ValidateKhmer, and tesseract::ValidateMyanmar.

IsSubscriptScript()

bool tesseract::Validator::IsSubscriptScript ( ) const

protected

Definition at line 198 of file validator.cpp.

                                        {
  return script_ == ViramaScript::kTelugu ||
         script_ == ViramaScript::kKannada ||
         script_ == ViramaScript::kJavanese ||
         script_ == ViramaScript::kMyanmar ||
         script_ == ViramaScript::kKhmer;
}

IsVedicAccent()

bool tesseract::Validator::IsVedicAccent ( char32  unicode )

staticprotected

Definition at line 191 of file validator.cpp.

                                            {
  return (0x1cd0 <= unicode && unicode < 0x1d00) ||
          (0xa8e0 <= unicode && unicode <= 0xa8f7) ||
          (0x951  <= unicode && unicode <= 0x954);
}

IsVirama()

bool tesseract::Validator::IsVirama ( char32  unicode )

staticprotected

Definition at line 180 of file validator.cpp.

                                       {
  return (kMinIndicUnicode <= unicode && unicode <= kMaxSinhalaUnicode &&
          (unicode & 0x7f) == 0x4d) ||
         unicode == kSinhalaVirama ||
         unicode == kJavaneseVirama ||
         unicode == kMyanmarVirama ||
         unicode == kKhmerVirama;
}

IsZeroWidthMark()

static bool tesseract::Validator::IsZeroWidthMark ( char32  ch )

inlinestatic

Definition at line 102 of file validator.h.

                       {

MostFrequentViramaScript()

ViramaScript tesseract::Validator::MostFrequentViramaScript ( const std::vector< char32 > &  utf32 )

staticprotected

Definition at line 143 of file validator.cpp.

                                    {
  std::unordered_map<int, int> histogram;
  for (char32 ch : utf32) {
    // Determine the codepage base. For the Indic scripts, Khmer and Javanese, it is
    // sufficient to divide by kIndicCodePageSize but Myanmar is all over the
    // unicode code space, so use its script id.
    int base = ch / kIndicCodePageSize;
    IcuErrorCode err;
    UScriptCode script_code = uscript_getScript(ch, err);
    if ((kMinIndicUnicode <= ch && ch <= kMaxJavaneseUnicode &&
         script_code != USCRIPT_COMMON) ||
        script_code == USCRIPT_MYANMAR) {
      if (script_code == USCRIPT_MYANMAR)
        base = static_cast<char32>(ViramaScript::kMyanmar) / kIndicCodePageSize;
      ++histogram[base];
    }
  }
  if (!histogram.empty()) {
    int base =
        std::max_element(histogram.begin(), histogram.end(), CmpPairSecond)
            ->first;
    char32 codebase = static_cast<char32>(base * kIndicCodePageSize);
    // Check for validity.
    if (codebase == static_cast<char32>(ViramaScript::kMyanmar) ||
        codebase == static_cast<char32>(ViramaScript::kJavanese) ||
        codebase == static_cast<char32>(ViramaScript::kKhmer) ||
        (static_cast<char32>(ViramaScript::kDevanagari) <= codebase &&
         codebase <= static_cast<char32>(ViramaScript::kSinhala))) {
      return static_cast<ViramaScript>(codebase);
    }
  }
  return ViramaScript::kNonVirama;
}

MoveResultsToDest()

void tesseract::Validator::MoveResultsToDest ( GraphemeNormMode  g_mode, std::vector< std::vector< char32 >> *  dest  )

protected

Definition at line 114 of file validator.cpp.

                                                                      {
  if (g_mode == GraphemeNormMode::kIndividualUnicodes) {
    // Append each element of the combined output_ that we made as a new vector
    // in dest.
    dest->reserve(dest->size() + output_.size());
    for (char32 ch : output_) dest->push_back({ch});
  } else if (g_mode == GraphemeNormMode::kGlyphSplit) {
    // Append all the parts_ that we made onto dest.
    std::move(parts_.begin(), parts_.end(), std::back_inserter(*dest));
  } else if (g_mode == GraphemeNormMode::kCombined || dest->empty()) {
    // Append the combined output_ that we made onto dest as one new vector.
    dest->push_back(std::vector<char32>());
    output_.swap(dest->back());
  } else {  // kNone.
    // Append the combined output_ that we made onto the last existing element
    // of dest.
    dest->back().insert(dest->back().end(), output_.begin(), output_.end());
  }
}

MultiCodePart()

void tesseract::Validator::MultiCodePart ( unsigned  length )

inlineprotected

Definition at line 196 of file validator.h.

ScriptValidator()

std::unique_ptr< Validator > tesseract::Validator::ScriptValidator ( ViramaScript  script, bool  report_errors  )

staticprotected

Definition at line 71 of file validator.cpp.

                                                                          {
  switch (script) {
    case ViramaScript::kNonVirama:
      return std::unique_ptr<Validator>(
          new ValidateGrapheme(script, report_errors));
    case ViramaScript::kJavanese:
      return std::unique_ptr<Validator>(
          new ValidateJavanese(script, report_errors));
    case ViramaScript::kMyanmar:
      return std::unique_ptr<Validator>(
          new ValidateMyanmar(script, report_errors));
    case ViramaScript::kKhmer:
      return std::unique_ptr<Validator>(
          new ValidateKhmer(script, report_errors));
    default:
      return std::unique_ptr<Validator>(
          new ValidateIndic(script, report_errors));
  }
}

UnicodeToCharClass()

virtual CharClass tesseract::Validator::UnicodeToCharClass ( char32  ch ) const

protectedpure virtual

Implemented in tesseract::ValidateJavanese, tesseract::ValidateIndic, tesseract::ValidateGrapheme, tesseract::ValidateKhmer, and tesseract::ValidateMyanmar.

UseMultiCode()

bool tesseract::Validator::UseMultiCode ( unsigned  length )

inlineprotected

Definition at line 210 of file validator.h.

ValidateCleanAndSegment()

bool tesseract::Validator::ValidateCleanAndSegment ( GraphemeNormMode  g_mode, bool  report_errors, const std::vector< char32 > &  src, std::vector< std::vector< char32 >> *  dest  )

static

Definition at line 40 of file validator.cpp.

                                        {
  ValidateGrapheme g_validator(ViramaScript::kNonVirama, report_errors);
  std::vector<std::vector<char32>> graphemes;
  ViramaScript script = MostFrequentViramaScript(src);
  bool success = true;
  if (script == ViramaScript::kNonVirama) {
    // The grapheme segmenter's maximum segmentation is the grapheme unit, so
    // up the mode by 1 to get the desired effect.
    if (g_mode == GraphemeNormMode::kCombined)
      g_mode = GraphemeNormMode::kGlyphSplit;
    else if (g_mode == GraphemeNormMode::kGlyphSplit)
      g_mode = GraphemeNormMode::kIndividualUnicodes;
    // Just do grapheme segmentation.
    success = g_validator.ValidateCleanAndSegmentInternal(g_mode, src, dest);
  } else {
    success = g_validator.ValidateCleanAndSegmentInternal(
        GraphemeNormMode::kGlyphSplit, src, &graphemes);
    std::unique_ptr<Validator> validator(
        ScriptValidator(script, report_errors));
    for (const auto& grapheme : graphemes) {
      if (!validator->ValidateCleanAndSegmentInternal(g_mode, grapheme, dest)) {
        success = false;
      }
    }
  }
  return success;
}

ValidateCleanAndSegmentInternal()

bool tesseract::Validator::ValidateCleanAndSegmentInternal ( GraphemeNormMode  g_mode, const std::vector< char32 > &  src, std::vector< std::vector< char32 >> *  dest  )

protected

Definition at line 97 of file validator.cpp.

                                        {
  Clear();
  ComputeClassCodes(src);
  bool success = true;
  for (codes_used_ = 0; codes_used_ < codes_.size();) {
    if (!ConsumeGraphemeIfValid()) {
      success = false;
      ++codes_used_;
    }
  }
  MoveResultsToDest(g_mode, dest);
  return success;
}

Member Data Documentation

codes_

std::vector<IndicPair> tesseract::Validator::codes_

protected

Definition at line 246 of file validator.h.

codes_used_

unsigned tesseract::Validator::codes_used_

protected

Definition at line 252 of file validator.h.

kIndicCodePageSize

const int tesseract::Validator::kIndicCodePageSize = 128

staticprotected

Definition at line 228 of file validator.h.

kInvalid

const char32 tesseract::Validator::kInvalid = 0xfffd

static

Definition at line 114 of file validator.h.

kJavaneseVirama

const char32 tesseract::Validator::kJavaneseVirama = 0xa9c0

staticprotected

Definition at line 240 of file validator.h.

kKhmerVirama

const char32 tesseract::Validator::kKhmerVirama = 0x17d2

staticprotected

Definition at line 238 of file validator.h.

kLeftToRightMark

const char32 tesseract::Validator::kLeftToRightMark = 0x200E

static

Definition at line 112 of file validator.h.

kMaxJavaneseUnicode

const char32 tesseract::Validator::kMaxJavaneseUnicode = 0xa9df

staticprotected

Definition at line 241 of file validator.h.

kMaxSinhalaUnicode

const char32 tesseract::Validator::kMaxSinhalaUnicode = 0xdff

staticprotected

Definition at line 232 of file validator.h.

kMaxViramaScriptUnicode

const char32 tesseract::Validator::kMaxViramaScriptUnicode = 0x17ff

staticprotected

Definition at line 234 of file validator.h.

kMinIndicUnicode

const char32 tesseract::Validator::kMinIndicUnicode = 0x900

staticprotected

Definition at line 230 of file validator.h.

kMyanmarVirama

const char32 tesseract::Validator::kMyanmarVirama = 0x1039

staticprotected

Definition at line 237 of file validator.h.

kRightToLeftMark

const char32 tesseract::Validator::kRightToLeftMark = 0x200F

static

Definition at line 113 of file validator.h.

kSinhalaVirama

const char32 tesseract::Validator::kSinhalaVirama = 0xdca

staticprotected

Definition at line 236 of file validator.h.

kZeroWidthJoiner

const char32 tesseract::Validator::kZeroWidthJoiner = 0x200D

static

Definition at line 111 of file validator.h.

kZeroWidthNonJoiner

const char32 tesseract::Validator::kZeroWidthNonJoiner = 0x200C

static

Definition at line 110 of file validator.h.

kZeroWidthSpace

const char32 tesseract::Validator::kZeroWidthSpace = 0x200B

static

Definition at line 109 of file validator.h.

output_

std::vector<char32> tesseract::Validator::output_

protected

Definition at line 250 of file validator.h.

output_used_

unsigned tesseract::Validator::output_used_

protected

Definition at line 254 of file validator.h.

parts_

std::vector<std::vector<char32> > tesseract::Validator::parts_

protected

Definition at line 248 of file validator.h.

report_errors_

bool tesseract::Validator::report_errors_

protected

Definition at line 256 of file validator.h.

script_

ViramaScript tesseract::Validator::script_

protected

Definition at line 244 of file validator.h.

---

The documentation for this class was generated from the following files:

• /tesseract/src/training/validator.h
• /tesseract/src/training/validator.cpp