unicodetext.h File Reference

#include <stddef.h>
#include <iterator>
#include <string>
#include <utility>
#include "syntaxnet/base.h"

Go to the source code of this file.

Classes
class	UnicodeText
class	UnicodeText::const_iterator
class	UnicodeText::const_reverse_iterator

Macros
#define	arraysize(array)   (sizeof(ArraySizeHelper(array)))

Typedefs
typedef pair< UnicodeText::const_iterator, UnicodeText::const_iterator >	UnicodeTextRange

Functions
bool	operator== (const UnicodeText &lhs, const UnicodeText &rhs)
bool	operator!= (const UnicodeText &lhs, const UnicodeText &rhs)
bool	UnicodeTextRangeIsEmpty (const UnicodeTextRange &r)
UnicodeText	MakeUnicodeTextAcceptingOwnership (char *utf8_buffer, int byte_length, int byte_capacity)
UnicodeText	MakeUnicodeTextWithoutAcceptingOwnership (const char *utf8_buffer, int byte_length)
UnicodeText	UTF8ToUnicodeText (const char *utf8_buf, int len, bool do_copy)
UnicodeText	UTF8ToUnicodeText (const string &utf_string, bool do_copy)
UnicodeText	UTF8ToUnicodeText (const char *utf8_buf, int len)
UnicodeText	UTF8ToUnicodeText (const string &utf8_string)
string	UnicodeTextToUTF8 (const UnicodeText &t)
template<typename T , size_t N>
char(&	ArraySizeHelper (T(&array)[N]))[N]
string	CodepointString (const UnicodeText &t)

Macro Definition Documentation

arraysize

#define arraysize

(

array

)

(sizeof(ArraySizeHelper(array)))

Definition at line 470 of file unicodetext.h.

Typedef Documentation

UnicodeTextRange

typedef pair<UnicodeText::const_iterator, UnicodeText::const_iterator> UnicodeTextRange

Definition at line 387 of file unicodetext.h.

Function Documentation

ArraySizeHelper()

template<typename T , size_t N>

char(& ArraySizeHelper

(

T(&)

array[N]

)

)[N]

CodepointString()

string CodepointString

(

const UnicodeText &

t

)

Definition at line 502 of file unicodetext.cc.

                                             {
  string s;
  UnicodeText::const_iterator it = t.begin(), end = t.end();
  while (it != end) tensorflow::strings::Appendf(&s, "%X ", *it++);
  return s;
}

MakeUnicodeTextAcceptingOwnership()

UnicodeText MakeUnicodeTextAcceptingOwnership ( char *  utf8_buffer, int  byte_length, int  byte_capacity  )

inline

Definition at line 405 of file unicodetext.h.

                                                           {
  return UnicodeText().TakeOwnershipOfUTF8(
      utf8_buffer, byte_length, byte_capacity);
}

MakeUnicodeTextWithoutAcceptingOwnership()

UnicodeText MakeUnicodeTextWithoutAcceptingOwnership ( const char *  utf8_buffer, int  byte_length  )

inline

Definition at line 415 of file unicodetext.h.

                                              {
  return UnicodeText().PointToUTF8(utf8_buffer, byte_length);
}

operator!=()

bool operator!= ( const UnicodeText &  lhs, const UnicodeText &  rhs  )

inline

Definition at line 380 of file unicodetext.h.

                                                                       {
  return !(lhs == rhs);
}

operator==()

bool operator==	(	const UnicodeText &	lhs,
		const UnicodeText &	rhs
	)

Definition at line 375 of file unicodetext.cc.

                                                                {
  if (&lhs == &rhs) return true;
  if (lhs.repr_.size_ != rhs.repr_.size_) return false;
  return memcmp(lhs.repr_.data_, rhs.repr_.data_, lhs.repr_.size_) == 0;
}

UnicodeTextRangeIsEmpty()

bool UnicodeTextRangeIsEmpty ( const UnicodeTextRange &  r )

inline

Definition at line 389 of file unicodetext.h.

                                                               {
  return r.first == r.second;
}

UnicodeTextToUTF8()

string UnicodeTextToUTF8 ( const UnicodeText &  t )

inline

Definition at line 461 of file unicodetext.h.

                                                      {
  return string(t.utf8_data(), t.utf8_length());
}

UTF8ToUnicodeText() [1/4]

UnicodeText UTF8ToUnicodeText ( const char *  utf8_buf, int  len  )

inline

Definition at line 452 of file unicodetext.h.

                                                                    {
  return UTF8ToUnicodeText(utf8_buf, len, true);
}

UTF8ToUnicodeText() [2/4]

UnicodeText UTF8ToUnicodeText ( const char *  utf8_buf, int  len, bool  do_copy  )

inline

Definition at line 437 of file unicodetext.h.

                                                   {
  UnicodeText t;
  if (do_copy) {
    t.CopyUTF8(utf8_buf, len);
  } else {
    t.PointToUTF8(utf8_buf, len);
  }
  return t;
}

UTF8ToUnicodeText() [3/4]

UnicodeText UTF8ToUnicodeText ( const string &  utf8_string )

inline

Definition at line 455 of file unicodetext.h.

                                                                {
  return UTF8ToUnicodeText(utf8_string, true);
}

UTF8ToUnicodeText() [4/4]

UnicodeText UTF8ToUnicodeText ( const string &  utf_string, bool  do_copy  )

inline

Definition at line 448 of file unicodetext.h.

                                                                             {
  return UTF8ToUnicodeText(utf_string.data(), utf_string.size(), do_copy);
}