#include <fullyconnected.h>

Inheritance diagram for tesseract::FullyConnected:

Public Member Functions
	FullyConnected (const STRING &name, int ni, int no, NetworkType type)

	~FullyConnected () override=default

StaticShape	OutputShape (const StaticShape &input_shape) const override

STRING	spec () const override

void	ChangeType (NetworkType type)

void	SetEnableTraining (TrainingState state) override

int	InitWeights (float range, TRand *randomizer) override

int	RemapOutputs (int old_no, const std::vector< int > &code_map) override

void	ConvertToInt () override

void	DebugWeights () override

bool	Serialize (TFile *fp) const override

bool	DeSerialize (TFile *fp) override

void	Forward (bool debug, const NetworkIO &input, const TransposedArray input_transpose, NetworkScratch scratch, NetworkIO *output) override

void	SetupForward (const NetworkIO &input, const TransposedArray *input_transpose)

void	ForwardTimeStep (int t, double *output_line)

void	ForwardTimeStep (const double d_input, int t, double output_line)

void	ForwardTimeStep (const int8_t i_input, int t, double output_line)

bool	Backward (bool debug, const NetworkIO &fwd_deltas, NetworkScratch scratch, NetworkIO back_deltas) override

void	BackwardTimeStep (const NetworkIO &fwd_deltas, int t, double curr_errors, TransposedArray errors_t, double *backprop)

void	FinishBackward (const TransposedArray &errors_t)

void	Update (float learning_rate, float momentum, float adam_beta, int num_samples) override

void	CountAlternators (const Network &other, double same, double changed) const override

Public Member Functions inherited from tesseract::Network
	Network ()

	Network (NetworkType type, const STRING &name, int ni, int no)

virtual	~Network ()=default

NetworkType	type () const

bool	IsTraining () const

bool	needs_to_backprop () const

int	num_weights () const

int	NumInputs () const

int	NumOutputs () const

virtual StaticShape	InputShape () const

const STRING &	name () const

bool	TestFlag (NetworkFlags flag) const

virtual bool	IsPlumbingType () const

virtual void	SetNetworkFlags (uint32_t flags)

virtual void	SetRandomizer (TRand *randomizer)

virtual bool	SetupNeedsBackprop (bool needs_backprop)

virtual int	XScaleFactor () const

virtual void	CacheXScaleFactor (int factor)

void	DisplayForward (const NetworkIO &matrix)

void	DisplayBackward (const NetworkIO &matrix)

Protected Attributes
WeightMatrix	weights_

TransposedArray	source_t_

const TransposedArray *	external_source_

NetworkIO	acts_

bool	int_mode_

Protected Attributes inherited from tesseract::Network
NetworkType	type_

TrainingState	training_

bool	needs_to_backprop_

int32_t	network_flags_

int32_t	ni_

int32_t	no_

int32_t	num_weights_

STRING	name_

ScrollView *	forward_win_

ScrollView *	backward_win_

TRand *	randomizer_

Additional Inherited Members
Static Public Member Functions inherited from tesseract::Network
static Network *	CreateFromFile (TFile *fp)

static void	ClearWindow (bool tess_coords, const char window_name, int width, int height, ScrollView *window)

static int	DisplayImage (Pix pix, ScrollView window)

Protected Member Functions inherited from tesseract::Network
double	Random (double range)

Detailed Description

Definition at line 28 of file fullyconnected.h.

Constructor & Destructor Documentation

◆ FullyConnected()

tesseract::FullyConnected::FullyConnected	(	const STRING &	name,
		int	ni,
		int	no,
		NetworkType	type
	)

Definition at line 38 of file fullyconnected.cpp.

40 : Network(type, name, ni, no), external_source_(nullptr), int_mode_(false) {

41 }

◆ ~FullyConnected()

tesseract::FullyConnected::~FullyConnected ( )

overridedefault

Member Function Documentation

◆ Backward()

bool tesseract::FullyConnected::Backward	(	bool	debug,
		const NetworkIO &	fwd_deltas,
		NetworkScratch *	scratch,
		NetworkIO *	back_deltas
	)

overridevirtual

Implements tesseract::Network.

Definition at line 220 of file fullyconnected.cpp.

                                                       {
   if (debug) DisplayBackward(fwd_deltas);
   back_deltas->Resize(fwd_deltas, ni_);
   GenericVector<NetworkScratch::FloatVec> errors;
   errors.init_to_size(kNumThreads, NetworkScratch::FloatVec());
   for (int i = 0; i < kNumThreads; ++i) errors[i].Init(no_, scratch);
   GenericVector<NetworkScratch::FloatVec> temp_backprops;
   if (needs_to_backprop_) {
     temp_backprops.init_to_size(kNumThreads, NetworkScratch::FloatVec());
     for (int i = 0; i < kNumThreads; ++i) temp_backprops[i].Init(ni_, scratch);
   }
   int width = fwd_deltas.Width();
   NetworkScratch::GradientStore errors_t;
   errors_t.Init(no_, width, scratch);
 #ifdef _OPENMP
 #pragma omp parallel for num_threads(kNumThreads)
   for (int t = 0; t < width; ++t) {
     int thread_id = omp_get_thread_num();
 #else
   for (int t = 0; t < width; ++t) {
     int thread_id = 0;
 #endif
     double* backprop = nullptr;
     if (needs_to_backprop_) backprop = temp_backprops[thread_id];
     double* curr_errors = errors[thread_id];
     BackwardTimeStep(fwd_deltas, t, curr_errors, errors_t.get(), backprop);
     if (backprop != nullptr) {
       back_deltas->WriteTimeStep(t, backprop);
     }
   }
   FinishBackward(*errors_t.get());
   if (needs_to_backprop_) {
     back_deltas->ZeroInvalidElements();
 #if DEBUG_DETAIL > 0
     tprintf("F Backprop:%s\n", name_.c_str());
     back_deltas->Print(10);
 #endif
     return true;
   }
   return false;  // No point going further back.
 }

◆ BackwardTimeStep()

void tesseract::FullyConnected::BackwardTimeStep	(	const NetworkIO &	fwd_deltas,
		int	t,
		double *	curr_errors,
		TransposedArray *	errors_t,
		double *	backprop
	)

Definition at line 264 of file fullyconnected.cpp.

                                                         {
   if (type_ == NT_TANH)
     acts_.FuncMultiply<GPrime>(fwd_deltas, t, curr_errors);
   else if (type_ == NT_LOGISTIC)
     acts_.FuncMultiply<FPrime>(fwd_deltas, t, curr_errors);
   else if (type_ == NT_POSCLIP)
     acts_.FuncMultiply<ClipFPrime>(fwd_deltas, t, curr_errors);
   else if (type_ == NT_SYMCLIP)
     acts_.FuncMultiply<ClipGPrime>(fwd_deltas, t, curr_errors);
   else if (type_ == NT_RELU)
     acts_.FuncMultiply<ReluPrime>(fwd_deltas, t, curr_errors);
   else if (type_ == NT_SOFTMAX || type_ == NT_SOFTMAX_NO_CTC ||
            type_ == NT_LINEAR)
     fwd_deltas.ReadTimeStep(t, curr_errors);  // fwd_deltas are the errors.
   else
     ASSERT_HOST("Invalid fully-connected type!" == nullptr);
   // Generate backprop only if needed by the lower layer.
   if (backprop != nullptr) weights_.VectorDotMatrix(curr_errors, backprop);
   errors_t->WriteStrided(t, curr_errors);
 }

◆ ChangeType()

void tesseract::FullyConnected::ChangeType ( NetworkType type )

inline

Definition at line 60 of file fullyconnected.h.

                                     {
     type_ = type;
   }

◆ ConvertToInt()

void tesseract::FullyConnected::ConvertToInt ( )

overridevirtual

Reimplemented from tesseract::Network.

Definition at line 95 of file fullyconnected.cpp.

                                   {
   weights_.ConvertToInt();
 }

◆ CountAlternators()

void tesseract::FullyConnected::CountAlternators	(	const Network &	other,
		double *	same,
		double *	changed
	)		const

overridevirtual

Reimplemented from tesseract::Network.

Definition at line 305 of file fullyconnected.cpp.

                                                              {
   ASSERT_HOST(other.type() == type_);
   const auto* fc = static_cast<const FullyConnected*>(&other);
   weights_.CountAlternators(fc->weights_, same, changed);
 }

◆ DebugWeights()

void tesseract::FullyConnected::DebugWeights ( )

overridevirtual

Implements tesseract::Network.

Definition at line 100 of file fullyconnected.cpp.

                                   {
   weights_.Debug2D(name_.c_str());
 }

◆ DeSerialize()

bool tesseract::FullyConnected::DeSerialize ( TFile * fp )

overridevirtual

Implements tesseract::Network.

Definition at line 112 of file fullyconnected.cpp.

                                           {
   return weights_.DeSerialize(IsTraining(), fp);
 }

◆ FinishBackward()

void tesseract::FullyConnected::FinishBackward ( const TransposedArray & errors_t )

Definition at line 288 of file fullyconnected.cpp.

                                                                    {
   if (external_source_ == nullptr)
     weights_.SumOuterTransposed(errors_t, source_t_, true);
   else
     weights_.SumOuterTransposed(errors_t, *external_source_, true);
 }

◆ Forward()

void tesseract::FullyConnected::Forward	(	bool	debug,
		const NetworkIO &	input,
		const TransposedArray *	input_transpose,
		NetworkScratch *	scratch,
		NetworkIO *	output
	)

overridevirtual

Implements tesseract::Network.

Definition at line 118 of file fullyconnected.cpp.

                                                                          {
   int width = input.Width();
   if (type_ == NT_SOFTMAX)
     output->ResizeFloat(input, no_);
   else
     output->Resize(input, no_);
   SetupForward(input, input_transpose);
   GenericVector<NetworkScratch::FloatVec> temp_lines;
   temp_lines.init_to_size(kNumThreads, NetworkScratch::FloatVec());
   GenericVector<NetworkScratch::FloatVec> curr_input;
   curr_input.init_to_size(kNumThreads, NetworkScratch::FloatVec());
   for (int i = 0; i < kNumThreads; ++i) {
     temp_lines[i].Init(no_, scratch);
     curr_input[i].Init(ni_, scratch);
   }
 #ifdef _OPENMP
 #pragma omp parallel for num_threads(kNumThreads)
   for (int t = 0; t < width; ++t) {
     // Thread-local pointer to temporary storage.
     int thread_id = omp_get_thread_num();
 #else
   for (int t = 0; t < width; ++t) {
     // Thread-local pointer to temporary storage.
     int thread_id = 0;
 #endif
     double* temp_line = temp_lines[thread_id];
     if (input.int_mode()) {
       ForwardTimeStep(input.i(t), t, temp_line);
     } else {
       input.ReadTimeStep(t, curr_input[thread_id]);
       ForwardTimeStep(curr_input[thread_id], t, temp_line);
     }
     output->WriteTimeStep(t, temp_line);
     if (IsTraining() && type_ != NT_SOFTMAX) {
       acts_.CopyTimeStepFrom(t, *output, t);
     }
   }
   // Zero all the elements that are in the padding around images that allows
   // multiple different-sized images to exist in a single array.
   // acts_ is only used if this is not a softmax op.
   if (IsTraining() && type_ != NT_SOFTMAX) {
     acts_.ZeroInvalidElements();
   }
   output->ZeroInvalidElements();
 #if DEBUG_DETAIL > 0
   tprintf("F Output:%s\n", name_.c_str());
   output->Print(10);
 #endif
   if (debug) DisplayForward(*output);
 }

◆ ForwardTimeStep() [1/3]

void tesseract::FullyConnected::ForwardTimeStep	(	const double *	d_input,
		int	t,
		double *	output_line
	)

Definition at line 202 of file fullyconnected.cpp.

                                                                  {
   // input is copied to source_ line-by-line for cache coherency.
   if (IsTraining() && external_source_ == nullptr)
     source_t_.WriteStrided(t, d_input);
   weights_.MatrixDotVector(d_input, output_line);
   ForwardTimeStep(t, output_line);
 }

◆ ForwardTimeStep() [2/3]

void tesseract::FullyConnected::ForwardTimeStep	(	const int8_t *	i_input,
		int	t,
		double *	output_line
	)

Definition at line 211 of file fullyconnected.cpp.

                                                                  {
   // input is copied to source_ line-by-line for cache coherency.
   weights_.MatrixDotVector(i_input, output_line);
   ForwardTimeStep(t, output_line);
 }

◆ ForwardTimeStep() [3/3]

void tesseract::FullyConnected::ForwardTimeStep	(	int	t,
		double *	output_line
	)

Definition at line 184 of file fullyconnected.cpp.

                                                                {
   if (type_ == NT_TANH) {
     FuncInplace<GFunc>(no_, output_line);
   } else if (type_ == NT_LOGISTIC) {
     FuncInplace<FFunc>(no_, output_line);
   } else if (type_ == NT_POSCLIP) {
     FuncInplace<ClipFFunc>(no_, output_line);
   } else if (type_ == NT_SYMCLIP) {
     FuncInplace<ClipGFunc>(no_, output_line);
   } else if (type_ == NT_RELU) {
     FuncInplace<Relu>(no_, output_line);
   } else if (type_ == NT_SOFTMAX || type_ == NT_SOFTMAX_NO_CTC) {
     SoftmaxInPlace(no_, output_line);
   } else if (type_ != NT_LINEAR) {
     ASSERT_HOST("Invalid fully-connected type!" == nullptr);
   }
 }

◆ InitWeights()

int tesseract::FullyConnected::InitWeights	(	float	range,
		TRand *	randomizer
	)

overridevirtual

Reimplemented from tesseract::Network.

Definition at line 76 of file fullyconnected.cpp.

                                                               {
   Network::SetRandomizer(randomizer);
   num_weights_ = weights_.InitWeightsFloat(no_, ni_ + 1, TestFlag(NF_ADAM),
                                            range, randomizer);
   return num_weights_;
 }

◆ OutputShape()

StaticShape tesseract::FullyConnected::OutputShape ( const StaticShape & input_shape ) const

overridevirtual

Reimplemented from tesseract::Network.

Definition at line 45 of file fullyconnected.cpp.

                                                                             {
   LossType loss_type = LT_NONE;
   if (type_ == NT_SOFTMAX)
     loss_type = LT_CTC;
   else if (type_ == NT_SOFTMAX_NO_CTC)
     loss_type = LT_SOFTMAX;
   else if (type_ == NT_LOGISTIC)
     loss_type = LT_LOGISTIC;
   StaticShape result(input_shape);
   result.set_depth(no_);
   result.set_loss_type(loss_type);
   return result;
 }

◆ RemapOutputs()

int tesseract::FullyConnected::RemapOutputs	(	int	old_no,
		const std::vector< int > &	code_map
	)

overridevirtual

Reimplemented from tesseract::Network.

Definition at line 86 of file fullyconnected.cpp.

                                                                            {
   if (type_ == NT_SOFTMAX && no_ == old_no) {
     num_weights_ = weights_.RemapOutputs(code_map);
     no_ = code_map.size();
   }
   return num_weights_;
 }

◆ Serialize()

bool tesseract::FullyConnected::Serialize ( TFile * fp ) const

overridevirtual

Reimplemented from tesseract::Network.

Definition at line 105 of file fullyconnected.cpp.

                                               {
   if (!Network::Serialize(fp)) return false;
   if (!weights_.Serialize(IsTraining(), fp)) return false;
   return true;
 }

◆ SetEnableTraining()

void tesseract::FullyConnected::SetEnableTraining ( TrainingState state )

overridevirtual

Reimplemented from tesseract::Network.

Definition at line 60 of file fullyconnected.cpp.

                                                           {
   if (state == TS_RE_ENABLE) {
     // Enable only from temp disabled.
     if (training_ == TS_TEMP_DISABLE) training_ = TS_ENABLED;
   } else if (state == TS_TEMP_DISABLE) {
     // Temp disable only from enabled.
     if (training_ == TS_ENABLED) training_ = state;
   } else {
     if (state == TS_ENABLED && training_ != TS_ENABLED)
       weights_.InitBackward();
     training_ = state;
   }
 }

◆ SetupForward()

void tesseract::FullyConnected::SetupForward	(	const NetworkIO &	input,
		const TransposedArray *	input_transpose
	)

Definition at line 172 of file fullyconnected.cpp.

                                                                           {
   // Softmax output is always float, so save the input type.
   int_mode_ = input.int_mode();
   if (IsTraining()) {
     acts_.Resize(input, no_);
     // Source_ is a transposed copy of input. It isn't needed if provided.
     external_source_ = input_transpose;
     if (external_source_ == nullptr) source_t_.ResizeNoInit(ni_, input.Width());
   }
 }

◆ spec()

STRING tesseract::FullyConnected::spec ( ) const

inlineoverridevirtual

Reimplemented from tesseract::Network.

Definition at line 37 of file fullyconnected.h.

                                {
     STRING spec;
     if (type_ == NT_TANH)
       spec.add_str_int("Ft", no_);
     else if (type_ == NT_LOGISTIC)
       spec.add_str_int("Fs", no_);
     else if (type_ == NT_RELU)
       spec.add_str_int("Fr", no_);
     else if (type_ == NT_LINEAR)
       spec.add_str_int("Fl", no_);
     else if (type_ == NT_POSCLIP)
       spec.add_str_int("Fp", no_);
     else if (type_ == NT_SYMCLIP)
       spec.add_str_int("Fs", no_);
     else if (type_ == NT_SOFTMAX)
       spec.add_str_int("Fc", no_);
     else
       spec.add_str_int("Fm", no_);
     return spec;
   }

◆ Update()

void tesseract::FullyConnected::Update	(	float	learning_rate,
		float	momentum,
		float	adam_beta,
		int	num_samples
	)

overridevirtual

Reimplemented from tesseract::Network.

Definition at line 297 of file fullyconnected.cpp.

                                                               {
   weights_.Update(learning_rate, momentum, adam_beta, num_samples);
 }

Member Data Documentation

◆ acts_

NetworkIO tesseract::FullyConnected::acts_

protected

Definition at line 126 of file fullyconnected.h.

◆ external_source_

const TransposedArray* tesseract::FullyConnected::external_source_

protected

Definition at line 124 of file fullyconnected.h.

◆ int_mode_

bool tesseract::FullyConnected::int_mode_

protected

Definition at line 129 of file fullyconnected.h.

◆ source_t_

TransposedArray tesseract::FullyConnected::source_t_

protected

Definition at line 121 of file fullyconnected.h.

◆ weights_

WeightMatrix tesseract::FullyConnected::weights_

protected

Definition at line 119 of file fullyconnected.h.

The documentation for this class was generated from the following files:

/tesseract/src/lstm/fullyconnected.h
/tesseract/src/lstm/fullyconnected.cpp

Public Member Functions

Protected Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ FullyConnected()

◆ ~FullyConnected()

Member Function Documentation

◆ Backward()

◆ BackwardTimeStep()

◆ ChangeType()

◆ ConvertToInt()

◆ CountAlternators()

◆ DebugWeights()

◆ DeSerialize()

◆ FinishBackward()

◆ Forward()

◆ ForwardTimeStep() [1/3]

◆ ForwardTimeStep() [2/3]

◆ ForwardTimeStep() [3/3]

◆ InitWeights()

◆ OutputShape()

◆ RemapOutputs()

◆ Serialize()

◆ SetEnableTraining()

◆ SetupForward()

◆ spec()

◆ Update()

Member Data Documentation

◆ acts_

◆ external_source_

◆ int_mode_

◆ source_t_

◆ weights_