normstrngs.h

Go to the documentation of this file.

/**********************************************************************
 * File:        normstrngs.h
 * Description: Utilities to normalize and manipulate UTF-32 and
 *              UTF-8 strings.
 * Author:      Ranjith Unnikrishnan
 * Created:     Thu July 4 2013
 *
 * (C) Copyright 2013, Google Inc.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * http://www.apache.org/licenses/LICENSE-2.0
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 **********************************************************************/

#ifndef TESSERACT_CCUTIL_NORMSTRNGS_H_
#define TESSERACT_CCUTIL_NORMSTRNGS_H_

#include <string>
#include <vector>

#include "validator.h"

namespace tesseract {

// The standard unicode normalizations.
enum class UnicodeNormMode {
  kNFD,
  kNFC,
  kNFKD,
  kNFKC,
};

// To normalize away differences in punctuation that are ambiguous, like
// curly quotes and different widths of dash.
enum class OCRNorm {
  kNone,
  kNormalize,
};

// To validate and normalize away some subtle differences that can occur in
// Indic scripts, eg ensuring that an explicit virama is always followed by
// a zero-width non-joiner.
enum class GraphemeNorm {
  kNone,
  kNormalize,
};

// Normalizes a UTF8 string according to the given modes. Returns true on
// success. If false is returned, some failure or invalidity was present, and
// the result string is produced on a "best effort" basis.
bool NormalizeUTF8String(UnicodeNormMode u_mode, OCRNorm ocr_normalize,
                         GraphemeNorm grapheme_normalize, const char* str8,
                         std::string* normalized);
// Normalizes a UTF8 string according to the given modes and splits into
// graphemes according to g_mode. Returns true on success. If false is returned,
// some failure or invalidity was present, and the result string is produced on
// a "best effort" basis.
bool NormalizeCleanAndSegmentUTF8(UnicodeNormMode u_mode, OCRNorm ocr_normalize,
                                  GraphemeNormMode g_mode, bool report_errors,
                                  const char* str8,
                                  std::vector<std::string>* graphemes);

// Applies just the OCR-specific normalizations and return the normalized char.
char32 OCRNormalize(char32 ch);

// Returns true if the OCRNormalized ch1 and ch2 are the same.
bool IsOCREquivalent(char32 ch1, char32 ch2);

// Returns true if the value lies in the range of valid unicodes.
bool IsValidCodepoint(const char32 ch);

// Returns true a code point has the White_Space Unicode property.
bool IsWhitespace(const char32 ch);
// Returns true if every char in the given (null-terminated) string has the
// White_Space Unicode property.
bool IsUTF8Whitespace(const char* text);

// Returns the length of bytes of the prefix of 'text' that have the White_Space
// unicode property.
unsigned int SpanUTF8Whitespace(const char* text);

// Returns the length of bytes of the prefix of 'text' that DO NOT have the
// White_Space unicode property.
unsigned int SpanUTF8NotWhitespace(const char* text);

// Returns true if the char is interchange valid i.e. no C0 or C1 control codes
// (other than CR LF HT FF) and no non-characters.
bool IsInterchangeValid(const char32 ch);
// Same as above but restricted to 7-bit ASCII.
bool IsInterchangeValid7BitAscii(const char32 ch);

// Convert a full-width UTF-8 string to half-width.
char32 FullwidthToHalfwidth(const char32 ch);

}  // namespace tesseract

#endif  // TESSERACT_CCUTIL_NORMSTRNGS_H_