commontraining.h File Reference

#include "cluster.h"
#include "commandlineflags.h"
#include "featdefs.h"
#include "intproto.h"
#include "oldlist.h"

Go to the source code of this file.

Classes
struct	LABELEDLISTNODE
struct	MERGE_CLASS_NODE

Namespaces
	tesseract

Typedefs
typedef struct LABELEDLISTNODE *	LABELEDLIST
using	MERGE_CLASS = MERGE_CLASS_NODE *

Functions
void	ParseArguments (int *argc, char ***argv)
ShapeTable *	tesseract::LoadShapeTable (const STRING &file_prefix)
void	tesseract::WriteShapeTable (const STRING &file_prefix, const ShapeTable &shape_table)
MasterTrainer *	tesseract::LoadTrainingData (int argc, const char *const *argv, bool replication, ShapeTable **shape_table, STRING *file_prefix)
const char *	GetNextFilename (int argc, const char *const *argv)
LABELEDLIST	FindList (LIST List, char *Label)
LABELEDLIST	NewLabeledList (const char *Label)
void	ReadTrainingSamples (const FEATURE_DEFS_STRUCT &feature_defs, const char *feature_name, int max_samples, UNICHARSET *unicharset, FILE *file, LIST *training_samples)
void	WriteTrainingSamples (const FEATURE_DEFS_STRUCT &FeatureDefs, char *Directory, LIST CharList, const char *program_feature_type)
void	FreeTrainingSamples (LIST CharList)
void	FreeLabeledList (LABELEDLIST LabeledList)
void	FreeLabeledClassList (LIST ClassListList)
CLUSTERER *	SetUpForClustering (const FEATURE_DEFS_STRUCT &FeatureDefs, LABELEDLIST CharSample, const char *program_feature_type)
LIST	RemoveInsignificantProtos (LIST ProtoList, bool KeepSigProtos, bool KeepInsigProtos, int N)
void	CleanUpUnusedData (LIST ProtoList)
void	MergeInsignificantProtos (LIST ProtoList, const char *label, CLUSTERER *Clusterer, CLUSTERCONFIG *Config)
MERGE_CLASS	FindClass (LIST List, const char *Label)
MERGE_CLASS	NewLabeledClass (const char *Label)
CLASS_STRUCT *	SetUpForFloat2Int (const UNICHARSET &unicharset, LIST LabeledClassList)
void	Normalize (float *Values)
void	FreeNormProtoList (LIST CharList)
void	AddToNormProtosList (LIST *NormProtoList, LIST ProtoList, char *CharName)
int	NumberOfProtos (LIST ProtoList, bool CountSigProtos, bool CountInsigProtos)
void	allocNormProtos ()

Variables
FEATURE_DEFS_STRUCT	feature_defs
CLUSTERCONFIG	Config

Typedef Documentation

LABELEDLIST

typedef struct LABELEDLISTNODE * LABELEDLIST

MERGE_CLASS

using MERGE_CLASS = MERGE_CLASS_NODE*

Definition at line 92 of file commontraining.h.

Function Documentation

AddToNormProtosList()

void AddToNormProtosList	(	LIST *	NormProtoList,
		LIST	ProtoList,
		char *	CharName
	)

Definition at line 836 of file commontraining.cpp.

{
  PROTOTYPE* Proto;
  LABELEDLIST LabeledProtoList;

  LabeledProtoList = NewLabeledList(CharName);
  iterate(ProtoList)
  {
    Proto = (PROTOTYPE *) first_node (ProtoList);
    LabeledProtoList->List = push(LabeledProtoList->List, Proto);
  }
  *NormProtoList = push(*NormProtoList, LabeledProtoList);
}

allocNormProtos()

void allocNormProtos

(

)

CleanUpUnusedData()

void CleanUpUnusedData

(

LIST

ProtoList

)

Definition at line 608 of file commontraining.cpp.

{
  PROTOTYPE* Prototype;

  iterate(ProtoList)
  {
    Prototype = (PROTOTYPE *) first_node (ProtoList);
    free(Prototype->Variance.Elliptical);
    Prototype->Variance.Elliptical = nullptr;
    free(Prototype->Magnitude.Elliptical);
    Prototype->Magnitude.Elliptical = nullptr;
    free(Prototype->Weight.Elliptical);
    Prototype->Weight.Elliptical = nullptr;
  }
}

FindClass()

MERGE_CLASS FindClass	(	LIST	List,
		const char *	Label
	)

Definition at line 691 of file commontraining.cpp.

                                                    {
  MERGE_CLASS MergeClass;

  iterate (List)
  {
    MergeClass = (MERGE_CLASS) first_node (List);
    if (strcmp (MergeClass->Label, Label) == 0)
      return (MergeClass);
  }
  return (nullptr);

} /* FindClass */

FindList()

LABELEDLIST FindList	(	LIST	List,
		char *	Label
	)

This routine searches through a list of labeled lists to find a list with the specified label. If a matching labeled list cannot be found, nullptr is returned.

Parameters

List	list to search
Label	label to search for

Returns

Labeled list with the specified label or nullptr.

Note

Globals: none

Definition at line 348 of file commontraining.cpp.

                                             {
  LABELEDLIST LabeledList;

  iterate (List)
  {
    LabeledList = (LABELEDLIST) first_node (List);
    if (strcmp (LabeledList->Label, Label) == 0)
      return (LabeledList);
  }
  return (nullptr);

} /* FindList */

FreeLabeledClassList()

void FreeLabeledClassList

(

LIST

ClassList

)

This routine deallocates all of the space allocated to the specified list of training samples.

Parameters

ClassList

list of all fonts in document

Returns

none

Note

Globals: none

Definition at line 724 of file commontraining.cpp.

                                          {
  MERGE_CLASS MergeClass;

  LIST nodes = ClassList;
  iterate(ClassList) /* iterate through all of the fonts */
  {
    MergeClass = (MERGE_CLASS) first_node (ClassList);
    free (MergeClass->Label);
    FreeClass(MergeClass->Class);
    delete MergeClass;
  }
  destroy(nodes);

} /* FreeLabeledClassList */

FreeLabeledList()

void FreeLabeledList

(

LABELEDLIST

LabeledList

)

This routine deallocates all of the memory consumed by a labeled list. It does not free any memory which may be consumed by the items in the list.

Parameters

LabeledList

labeled list to be freed

Note

Globals: none

Returns

none

Definition at line 489 of file commontraining.cpp.

                                              {
  destroy(LabeledList->List);
  free(LabeledList->Label);
  free(LabeledList);
}  /* FreeLabeledList */

FreeNormProtoList()

void FreeNormProtoList

(

LIST

CharList

)

Definition at line 820 of file commontraining.cpp.

{
  LABELEDLIST char_sample;

  LIST nodes = CharList;
  iterate(CharList) /* iterate through all of the fonts */
  {
    char_sample = (LABELEDLIST) first_node (CharList);
    FreeLabeledList (char_sample);
  }
  destroy(nodes);

}  // FreeNormProtoList

FreeTrainingSamples()

void FreeTrainingSamples

(

LIST

CharList

)

This routine deallocates all of the space allocated to the specified list of training samples.

Parameters

CharList

list of all fonts in document

Returns

none

Note

Globals: none

Definition at line 462 of file commontraining.cpp.

                                        {
  LABELEDLIST char_sample;
  FEATURE_SET FeatureSet;
  LIST FeatureList;

  LIST nodes = CharList;
  iterate(CharList) { /* iterate through all of the fonts */
    char_sample = (LABELEDLIST) first_node(CharList);
    FeatureList = char_sample->List;
    iterate(FeatureList) { /* iterate through all of the classes */
      FeatureSet = (FEATURE_SET) first_node(FeatureList);
      FreeFeatureSet(FeatureSet);
    }
    FreeLabeledList(char_sample);
  }
  destroy(nodes);
}  /* FreeTrainingSamples */

GetNextFilename()

const char* GetNextFilename	(	int	argc,
		const char *const *	argv
	)

This routine returns the next command line argument. If there are no remaining command line arguments, it returns nullptr. This routine should only be called after all option arguments have been parsed and removed with ParseArguments.

Globals:

• tessoptind defined by tessopt sys call

  Returns

  Next command line argument or nullptr.

Definition at line 331 of file commontraining.cpp.

                                                                {
  if (tessoptind < argc)
    return argv[tessoptind++];
  else
    return nullptr;
} /* GetNextFilename */

MergeInsignificantProtos()

void MergeInsignificantProtos	(	LIST	ProtoList,
		const char *	label,
		CLUSTERER *	Clusterer,
		CLUSTERCONFIG *	Config
	)

Definition at line 541 of file commontraining.cpp.

                                                            {
  PROTOTYPE* Prototype;
  bool debug = strcmp(FLAGS_test_ch.c_str(), label) == 0;

  LIST pProtoList = ProtoList;
  iterate(pProtoList) {
    Prototype = (PROTOTYPE *) first_node (pProtoList);
    if (Prototype->Significant || Prototype->Merged)
      continue;
    float best_dist = 0.125;
    PROTOTYPE* best_match = nullptr;
    // Find the nearest alive prototype.
    LIST list_it = ProtoList;
    iterate(list_it) {
      PROTOTYPE* test_p = (PROTOTYPE *) first_node (list_it);
      if (test_p != Prototype && !test_p->Merged) {
        float dist = ComputeDistance(Clusterer->SampleSize,
                                     Clusterer->ParamDesc,
                                     Prototype->Mean, test_p->Mean);
        if (dist < best_dist) {
          best_match = test_p;
          best_dist = dist;
        }
      }
    }
    if (best_match != nullptr && !best_match->Significant) {
      if (debug)
        tprintf("Merging red clusters (%d+%d) at %g,%g and %g,%g\n",
                best_match->NumSamples, Prototype->NumSamples,
                best_match->Mean[0], best_match->Mean[1],
                Prototype->Mean[0], Prototype->Mean[1]);
      best_match->NumSamples = MergeClusters(Clusterer->SampleSize,
                                             Clusterer->ParamDesc,
                                             best_match->NumSamples,
                                             Prototype->NumSamples,
                                             best_match->Mean,
                                             best_match->Mean, Prototype->Mean);
      Prototype->NumSamples = 0;
      Prototype->Merged = 1;
    } else if (best_match != nullptr) {
      if (debug)
        tprintf("Red proto at %g,%g matched a green one at %g,%g\n",
                Prototype->Mean[0], Prototype->Mean[1],
                best_match->Mean[0], best_match->Mean[1]);
      Prototype->Merged = 1;
    }
  }
  // Mark significant those that now have enough samples.
  int min_samples =
    static_cast<int32_t>(clusterconfig->MinSamples * Clusterer->NumChar);
  pProtoList = ProtoList;
  iterate(pProtoList) {
    Prototype = (PROTOTYPE *) first_node (pProtoList);
    // Process insignificant protos that do not match a green one
    if (!Prototype->Significant && Prototype->NumSamples >= min_samples &&
        !Prototype->Merged) {
      if (debug)
        tprintf("Red proto at %g,%g becoming green\n",
                Prototype->Mean[0], Prototype->Mean[1]);
      Prototype->Significant = true;
    }
  }
} /* MergeInsignificantProtos */

NewLabeledClass()

MERGE_CLASS NewLabeledClass

(

const char *

Label

)

Definition at line 705 of file commontraining.cpp.

                                               {
  MERGE_CLASS MergeClass;

  MergeClass = new MERGE_CLASS_NODE;
  MergeClass->Label = (char*)Emalloc (strlen (Label)+1);
  strcpy (MergeClass->Label, Label);
  MergeClass->Class = NewClass (MAX_NUM_PROTOS, MAX_NUM_CONFIGS);
  return (MergeClass);

} /* NewLabeledClass */

NewLabeledList()

LABELEDLIST NewLabeledList

(

const char *

Label

)

This routine allocates a new, empty labeled list and gives it the specified label.

Parameters

Label

label for new list

Returns

New, empty labeled list.

Note

Globals: none

Definition at line 369 of file commontraining.cpp.

                                              {
  LABELEDLIST LabeledList;

  LabeledList = (LABELEDLIST) Emalloc (sizeof (LABELEDLISTNODE));
  LabeledList->Label = (char*)Emalloc (strlen (Label)+1);
  strcpy (LabeledList->Label, Label);
  LabeledList->List = NIL_LIST;
  LabeledList->SampleCount = 0;
  LabeledList->font_sample_count = 0;
  return (LabeledList);

} /* NewLabeledList */

Normalize()

void Normalize

(

float *

Values

)

Definition at line 803 of file commontraining.cpp.

{
  float Slope;
  float Intercept;
  float Normalizer;

  Slope      = tan(Values [2] * 2 * M_PI);
  Intercept  = Values [1] - Slope * Values [0];
  Normalizer = 1 / sqrt (Slope * Slope + 1.0);

  Values [0] = Slope * Normalizer;
  Values [1] = - Normalizer;
  Values [2] = Intercept * Normalizer;
} // Normalize

NumberOfProtos()

int NumberOfProtos	(	LIST	ProtoList,
		bool	CountSigProtos,
		bool	CountInsigProtos
	)

Definition at line 854 of file commontraining.cpp.

                                          {
  int N = 0;
  iterate(ProtoList)
  {
    PROTOTYPE* Proto = (PROTOTYPE*)first_node(ProtoList);
    if ((Proto->Significant && CountSigProtos) ||
        (!Proto->Significant && CountInsigProtos))
      N++;
  }
  return(N);
}

ParseArguments()

void ParseArguments	(	int *	argc,
		char ***	argv
	)

This routine parses the command line arguments that were passed to the program and uses them to set relevant training-related global parameters.

Globals:

• Config current clustering parameters

  Parameters

  argc	number of command line arguments to parse
  argv	command line arguments

  Returns

  none

Definition at line 130 of file commontraining.cpp.

                                             {
  STRING usage;
  if (*argc) {
    usage += (*argv)[0];
    usage += " -v | --version | ";
    usage += (*argv)[0];
  }
  usage += " [.tr files ...]";
  tesseract::ParseCommandLineFlags(usage.c_str(), argc, argv, true);
  // Record the index of the first non-flag argument to 1, since we set
  // remove_flags to true when parsing the flags.
  tessoptind = 1;
  // Set some global values based on the flags.
  Config.MinSamples =
          std::max(0.0, std::min(1.0, double(FLAGS_clusterconfig_min_samples_fraction)));
  Config.MaxIllegal =
          std::max(0.0, std::min(1.0, double(FLAGS_clusterconfig_max_illegal)));
  Config.Independence =
          std::max(0.0, std::min(1.0, double(FLAGS_clusterconfig_independence)));
  Config.Confidence =
          std::max(0.0, std::min(1.0, double(FLAGS_clusterconfig_confidence)));
  // Set additional parameters from config file if specified.
  if (!FLAGS_configfile.empty()) {
    tesseract::ParamUtils::ReadParamsFile(
        FLAGS_configfile.c_str(),
        tesseract::SET_PARAM_CONSTRAINT_NON_INIT_ONLY,
        ccutil.params());
  }
}

ReadTrainingSamples()

void ReadTrainingSamples	(	const FEATURE_DEFS_STRUCT &	feature_definitions,
		const char *	feature_name,
		int	max_samples,
		UNICHARSET *	unicharset,
		FILE *	file,
		LIST *	training_samples
	)

This routine reads training samples from a file and places them into a data structure which organizes the samples by FontName and CharName. It then returns this data structure.

Parameters

file	open text file to read samples from
feature_definitions
feature_name
max_samples
unicharset
training_samples

Returns

none

Note

Globals: none

Definition at line 399 of file commontraining.cpp.

                                                             {
  char    buffer[2048];
  char    unichar[UNICHAR_LEN + 1];
  LABELEDLIST char_sample;
  FEATURE_SET feature_samples;
  CHAR_DESC char_desc;
  uint32_t feature_type =
    ShortNameToFeatureType(feature_definitions, feature_name);

  // Zero out the font_sample_count for all the classes.
  LIST it = *training_samples;
  iterate(it) {
    char_sample = reinterpret_cast<LABELEDLIST>(first_node(it));
    char_sample->font_sample_count = 0;
  }

  while (fgets(buffer, 2048, file) != nullptr) {
    if (buffer[0] == '\n')
      continue;

    sscanf(buffer, "%*s %s", unichar);
    if (unicharset != nullptr && !unicharset->contains_unichar(unichar)) {
      unicharset->unichar_insert(unichar);
      if (unicharset->size() > MAX_NUM_CLASSES) {
        tprintf("Error: Size of unicharset in training is "
                "greater than MAX_NUM_CLASSES\n");
        exit(1);
      }
    }
    char_sample = FindList(*training_samples, unichar);
    if (char_sample == nullptr) {
      char_sample = NewLabeledList(unichar);
      *training_samples = push(*training_samples, char_sample);
    }
    char_desc = ReadCharDescription(feature_definitions, file);
    feature_samples = char_desc->FeatureSets[feature_type];
    if (char_sample->font_sample_count < max_samples || max_samples <= 0) {
      char_sample->List = push(char_sample->List, feature_samples);
      char_sample->SampleCount++;
      char_sample->font_sample_count++;
    } else {
      FreeFeatureSet(feature_samples);
    }
    for (size_t i = 0; i < char_desc->NumFeatureSets; i++) {
      if (feature_type != i)
        FreeFeatureSet(char_desc->FeatureSets[i]);
    }
    free(char_desc);
  }
}  // ReadTrainingSamples

RemoveInsignificantProtos()

LIST RemoveInsignificantProtos	(	LIST	ProtoList,
		bool	KeepSigProtos,
		bool	KeepInsigProtos,
		int	N
	)

Definition at line 626 of file commontraining.cpp.

{
  LIST NewProtoList = NIL_LIST;
  LIST pProtoList;
  PROTOTYPE* Proto;
  PROTOTYPE* NewProto;
  int i;

  pProtoList = ProtoList;
  iterate(pProtoList)
  {
    Proto = (PROTOTYPE *) first_node (pProtoList);
    if ((Proto->Significant && KeepSigProtos) ||
        (!Proto->Significant && KeepInsigProtos))
    {
      NewProto = (PROTOTYPE *)Emalloc(sizeof(PROTOTYPE));

      NewProto->Mean = (float *)Emalloc(N * sizeof(float));
      NewProto->Significant = Proto->Significant;
      NewProto->Style = Proto->Style;
      NewProto->NumSamples = Proto->NumSamples;
      NewProto->Cluster = nullptr;
      NewProto->Distrib = nullptr;

      for (i=0; i < N; i++)
        NewProto->Mean[i] = Proto->Mean[i];
      if (Proto->Variance.Elliptical != nullptr) {
        NewProto->Variance.Elliptical = (float *)Emalloc(N * sizeof(float));
        for (i=0; i < N; i++)
          NewProto->Variance.Elliptical[i] = Proto->Variance.Elliptical[i];
      }
      else
        NewProto->Variance.Elliptical = nullptr;
      //---------------------------------------------
      if (Proto->Magnitude.Elliptical != nullptr) {
        NewProto->Magnitude.Elliptical = (float *)Emalloc(N * sizeof(float));
        for (i=0; i < N; i++)
          NewProto->Magnitude.Elliptical[i] = Proto->Magnitude.Elliptical[i];
      }
      else
        NewProto->Magnitude.Elliptical = nullptr;
      //------------------------------------------------
      if (Proto->Weight.Elliptical != nullptr) {
        NewProto->Weight.Elliptical = (float *)Emalloc(N * sizeof(float));
        for (i=0; i < N; i++)
          NewProto->Weight.Elliptical[i] = Proto->Weight.Elliptical[i];
      }
      else
        NewProto->Weight.Elliptical = nullptr;

      NewProto->TotalMagnitude = Proto->TotalMagnitude;
      NewProto->LogMagnitude = Proto->LogMagnitude;
      NewProtoList = push_last(NewProtoList, NewProto);
    }
  }
  FreeProtoList(&ProtoList);
  return (NewProtoList);
} /* RemoveInsignificantProtos */

SetUpForClustering()

CLUSTERER* SetUpForClustering	(	const FEATURE_DEFS_STRUCT &	FeatureDefs,
		LABELEDLIST	char_sample,
		const char *	program_feature_type
	)

This routine reads samples from a LABELEDLIST and enters those samples into a clusterer data structure. This data structure is then returned to the caller.

Parameters

char_sample	LABELEDLIST that holds all the feature information for a
FeatureDefs
program_feature_type	given character.

Returns

Pointer to new clusterer data structure.

Note

Globals: None

Definition at line 507 of file commontraining.cpp.

                                                                {
  uint16_t N;
  int i, j;
  float* Sample = nullptr;
  CLUSTERER *Clusterer;
  int32_t CharID;
  LIST FeatureList = nullptr;
  FEATURE_SET FeatureSet = nullptr;

  int32_t desc_index =
      ShortNameToFeatureType(FeatureDefs, program_feature_type);
  N = FeatureDefs.FeatureDesc[desc_index]->NumParams;
  Clusterer = MakeClusterer(N, FeatureDefs.FeatureDesc[desc_index]->ParamDesc);

  FeatureList = char_sample->List;
  CharID = 0;
  iterate(FeatureList) {
    FeatureSet = (FEATURE_SET) first_node(FeatureList);
    for (i = 0; i < FeatureSet->MaxNumFeatures; i++) {
      if (Sample == nullptr) Sample = (float*)Emalloc(N * sizeof(float));
      for (j = 0; j < N; j++)
        Sample[j] = FeatureSet->Features[i]->Params[j];
      MakeSample (Clusterer, Sample, CharID);
    }
    CharID++;
  }
  free(Sample);
  return Clusterer;

} /* SetUpForClustering */

SetUpForFloat2Int()

CLASS_STRUCT* SetUpForFloat2Int	(	const UNICHARSET &	unicharset,
		LIST	LabeledClassList
	)

Definition at line 740 of file commontraining.cpp.

                                                       {
  MERGE_CLASS MergeClass;
  CLASS_TYPE Class;
  int NumProtos;
  int NumConfigs;
  int NumWords;
  int i, j;
  float Values[3];
  PROTO NewProto;
  PROTO OldProto;
  BIT_VECTOR NewConfig;
  BIT_VECTOR OldConfig;

  //  printf("Float2Int ...\n");

  CLASS_STRUCT* float_classes = new CLASS_STRUCT[unicharset.size()];
  iterate(LabeledClassList)
  {
    UnicityTableEqEq<int>   font_set;
    MergeClass = (MERGE_CLASS) first_node (LabeledClassList);
    Class = &float_classes[unicharset.unichar_to_id(MergeClass->Label)];
    NumProtos = MergeClass->Class->NumProtos;
    NumConfigs = MergeClass->Class->NumConfigs;
    font_set.move(&MergeClass->Class->font_set);
    Class->NumProtos = NumProtos;
    Class->MaxNumProtos = NumProtos;
    Class->Prototypes = (PROTO) Emalloc (sizeof(PROTO_STRUCT) * NumProtos);
    for(i=0; i < NumProtos; i++)
    {
      NewProto = ProtoIn(Class, i);
      OldProto = ProtoIn(MergeClass->Class, i);
      Values[0] = OldProto->X;
      Values[1] = OldProto->Y;
      Values[2] = OldProto->Angle;
      Normalize(Values);
      NewProto->X = OldProto->X;
      NewProto->Y = OldProto->Y;
      NewProto->Length = OldProto->Length;
      NewProto->Angle = OldProto->Angle;
      NewProto->A = Values[0];
      NewProto->B = Values[1];
      NewProto->C = Values[2];
    }

    Class->NumConfigs = NumConfigs;
    Class->MaxNumConfigs = NumConfigs;
    Class->font_set.move(&font_set);
    Class->Configurations = (BIT_VECTOR*) Emalloc (sizeof(BIT_VECTOR) * NumConfigs);
    NumWords = WordsInVectorOfSize(NumProtos);
    for(i=0; i < NumConfigs; i++)
    {
      NewConfig = NewBitVector(NumProtos);
      OldConfig = MergeClass->Class->Configurations[i];
      for(j=0; j < NumWords; j++)
        NewConfig[j] = OldConfig[j];
      Class->Configurations[i] = NewConfig;
    }
  }
  return float_classes;
} // SetUpForFloat2Int

WriteTrainingSamples()

void WriteTrainingSamples	(	const FEATURE_DEFS_STRUCT &	FeatureDefs,
		char *	Directory,
		LIST	CharList,
		const char *	program_feature_type
	)

Variable Documentation

Config

CLUSTERCONFIG Config

Definition at line 95 of file commontraining.cpp.

feature_defs

FEATURE_DEFS_STRUCT feature_defs

Definition at line 96 of file commontraining.cpp.