tesseract  4.0.0-1-g2a2b
functions.h
Go to the documentation of this file.
1 // File: functions.h
3 // Description: Collection of function-objects used by the network layers.
4 // Author: Ray Smith
5 // Created: Fri Jun 20 10:45:37 PST 2014
6 //
7 // (C) Copyright 2014, Google Inc.
8 // Licensed under the Apache License, Version 2.0 (the "License");
9 // you may not use this file except in compliance with the License.
10 // You may obtain a copy of the License at
11 // http://www.apache.org/licenses/LICENSE-2.0
12 // Unless required by applicable law or agreed to in writing, software
13 // distributed under the License is distributed on an "AS IS" BASIS,
14 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 // See the License for the specific language governing permissions and
16 // limitations under the License.
18 
19 #ifndef TESSERACT_LSTM_FUNCTIONS_H_
20 #define TESSERACT_LSTM_FUNCTIONS_H_
21 
22 #include <cmath>
23 #include "helpers.h"
24 #include "tprintf.h"
25 
26 // Setting this to 1 or more causes massive dumps of debug data: weights,
27 // updates, internal calculations etc, and reduces the number of test iterations
28 // to a small number, so outputs can be diffed.
29 #define DEBUG_DETAIL 0
30 #if DEBUG_DETAIL > 0
31 #undef _OPENMP // Disable open mp to get the outputs in sync.
32 #endif
33 
34 namespace tesseract {
35 
36 // Size of static tables.
37 const int kTableSize = 4096;
38 // Scale factor for float arg to int index.
39 const double kScaleFactor = 256.0;
40 
41 extern double TanhTable[];
42 extern double LogisticTable[];
43 
44 // Non-linearity (sigmoid) functions with cache tables and clipping.
45 inline double Tanh(double x) {
46  if (x < 0.0) return -Tanh(-x);
47  if (x >= (kTableSize - 1) / kScaleFactor) return 1.0;
48  x *= kScaleFactor;
49  int index = static_cast<int>(floor(x));
50  if (TanhTable[index] == 0.0 && index > 0) {
51  // Generate the entry.
52  TanhTable[index] = tanh(index / kScaleFactor);
53  }
54  if (index == kTableSize - 1) return TanhTable[kTableSize - 1];
55  if (TanhTable[index + 1] == 0.0) {
56  // Generate the entry.
57  TanhTable[index + 1] = tanh((index + 1) / kScaleFactor);
58  }
59  double offset = x - index;
60  return TanhTable[index] * (1.0 - offset) + TanhTable[index + 1] * offset;
61 }
62 
63 inline double Logistic(double x) {
64  if (x < 0.0) return 1.0 - Logistic(-x);
65  if (x >= (kTableSize - 1) / kScaleFactor) return 1.0;
66  x *= kScaleFactor;
67  int index = static_cast<int>(floor(x));
68  if (LogisticTable[index] == 0.0) {
69  // Generate the entry.
70  LogisticTable[index] = 1.0 / (1.0 + exp(-index / kScaleFactor));
71  }
72  if (index == kTableSize - 1) return LogisticTable[kTableSize - 1];
73  if (LogisticTable[index + 1] == 0.0) {
74  // Generate the entry.
75  LogisticTable[index + 1] = 1.0 / (1.0 + exp(-(index + 1) / kScaleFactor));
76  }
77  double offset = x - index;
78  return LogisticTable[index] * (1.0 - offset) +
79  LogisticTable[index + 1] * offset;
80 }
81 
82 // Non-linearity (sigmoid) functions and their derivatives.
83 struct FFunc {
84  inline double operator()(double x) const { return Logistic(x); }
85 };
86 struct FPrime {
87  inline double operator()(double y) const { return y * (1.0 - y); }
88 };
89 struct ClipFFunc {
90  inline double operator()(double x) const {
91  if (x <= 0.0) return 0.0;
92  if (x >= 1.0) return 1.0;
93  return x;
94  }
95 };
96 struct ClipFPrime {
97  inline double operator()(double y) const {
98  return 0.0 < y && y < 1.0 ? 1.0 : 0.0;
99  }
100 };
101 struct Relu {
102  inline double operator()(double x) const {
103  if (x <= 0.0) return 0.0;
104  return x;
105  }
106 };
107 struct ReluPrime {
108  inline double operator()(double y) const { return 0.0 < y ? 1.0 : 0.0; }
109 };
110 struct GFunc {
111  inline double operator()(double x) const { return Tanh(x); }
112 };
113 struct GPrime {
114  inline double operator()(double y) const { return 1.0 - y * y; }
115 };
116 struct ClipGFunc {
117  inline double operator()(double x) const {
118  if (x <= -1.0) return -1.0;
119  if (x >= 1.0) return 1.0;
120  return x;
121  }
122 };
123 struct ClipGPrime {
124  inline double operator()(double y) const {
125  return -1.0 < y && y < 1.0 ? 1.0 : 0.0;
126  }
127 };
128 struct HFunc {
129  inline double operator()(double x) const { return Tanh(x); }
130 };
131 struct HPrime {
132  inline double operator()(double y) const {
133  double u = Tanh(y);
134  return 1.0 - u * u;
135  }
136 };
137 struct UnityFunc {
138  inline double operator()(double x) const { return 1.0; }
139 };
140 struct IdentityFunc {
141  inline double operator()(double x) const { return x; }
142 };
143 
144 // Applies Func in-place to inout, of size n.
145 template <class Func>
146 inline void FuncInplace(int n, double* inout) {
147  Func f;
148  for (int i = 0; i < n; ++i) {
149  inout[i] = f(inout[i]);
150  }
151 }
152 // Applies Func to u and multiplies the result by v component-wise,
153 // putting the product in out, all of size n.
154 template <class Func>
155 inline void FuncMultiply(const double* u, const double* v, int n, double* out) {
156  Func f;
157  for (int i = 0; i < n; ++i) {
158  out[i] = f(u[i]) * v[i];
159  }
160 }
161 // Applies the Softmax function in-place to inout, of size n.
162 template <typename T>
163 inline void SoftmaxInPlace(int n, T* inout) {
164  if (n <= 0) return;
165  // A limit on the negative range input to exp to guarantee non-zero output.
166  const T kMaxSoftmaxActivation = 86.0f;
167 
168  T max_output = inout[0];
169  for (int i = 1; i < n; i++) {
170  T output = inout[i];
171  if (output > max_output) max_output = output;
172  }
173  T prob_total = 0.0;
174  for (int i = 0; i < n; i++) {
175  T prob = inout[i] - max_output;
176  prob = exp(ClipToRange(prob, -kMaxSoftmaxActivation, static_cast<T>(0)));
177  prob_total += prob;
178  inout[i] = prob;
179  }
180  if (prob_total > 0.0) {
181  for (int i = 0; i < n; i++) inout[i] /= prob_total;
182  }
183 }
184 
185 // Copies n values of the given src vector to dest.
186 inline void CopyVector(int n, const double* src, double* dest) {
187  memcpy(dest, src, n * sizeof(dest[0]));
188 }
189 
190 // Adds n values of the given src vector to dest.
191 inline void AccumulateVector(int n, const double* src, double* dest) {
192  for (int i = 0; i < n; ++i) dest[i] += src[i];
193 }
194 
195 // Multiplies n values of inout in-place element-wise by the given src vector.
196 inline void MultiplyVectorsInPlace(int n, const double* src, double* inout) {
197  for (int i = 0; i < n; ++i) inout[i] *= src[i];
198 }
199 
200 // Multiplies n values of u by v, element-wise, accumulating to out.
201 inline void MultiplyAccumulate(int n, const double* u, const double* v,
202  double* out) {
203  for (int i = 0; i < n; i++) {
204  out[i] += u[i] * v[i];
205  }
206 }
207 
208 // Sums the given 5 n-vectors putting the result into sum.
209 inline void SumVectors(int n, const double* v1, const double* v2,
210  const double* v3, const double* v4, const double* v5,
211  double* sum) {
212  for (int i = 0; i < n; ++i) {
213  sum[i] = v1[i] + v2[i] + v3[i] + v4[i] + v5[i];
214  }
215 }
216 
217 // Sets the given n-vector vec to 0.
218 template <typename T>
219 inline void ZeroVector(int n, T* vec) {
220  memset(vec, 0, n * sizeof(*vec));
221 }
222 
223 // Clips the given vector vec, of size n to [lower, upper].
224 template <typename T>
225 inline void ClipVector(int n, T lower, T upper, T* vec) {
226  for (int i = 0; i < n; ++i) vec[i] = ClipToRange(vec[i], lower, upper);
227 }
228 
229 // Converts the given n-vector to a binary encoding of the maximum value,
230 // encoded as vector of nf binary values.
231 inline void CodeInBinary(int n, int nf, double* vec) {
232  if (nf <= 0 || n < nf) return;
233  int index = 0;
234  double best_score = vec[0];
235  for (int i = 1; i < n; ++i) {
236  if (vec[i] > best_score) {
237  best_score = vec[i];
238  index = i;
239  }
240  }
241  int mask = 1;
242  for (int i = 0; i < nf; ++i, mask *= 2) {
243  vec[i] = (index & mask) ? 1.0 : 0.0;
244  }
245 }
246 
247 } // namespace tesseract.
248 
249 #endif // TESSERACT_LSTM_FUNCTIONS_H_
double operator()(double x) const
Definition: functions.h:90
double operator()(double x) const
Definition: functions.h:138
double operator()(double y) const
Definition: functions.h:114
void FuncInplace(int n, double *inout)
Definition: functions.h:146
void CodeInBinary(int n, int nf, double *vec)
Definition: functions.h:231
void SoftmaxInPlace(int n, T *inout)
Definition: functions.h:163
double operator()(double x) const
Definition: functions.h:84
void CopyVector(int n, const double *src, double *dest)
Definition: functions.h:186
double LogisticTable[kTableSize]
Definition: functions.cpp:24
double operator()(double y) const
Definition: functions.h:87
void ClipVector(int n, T lower, T upper, T *vec)
Definition: functions.h:225
void MultiplyVectorsInPlace(int n, const double *src, double *inout)
Definition: functions.h:196
double operator()(double y) const
Definition: functions.h:97
void MultiplyAccumulate(int n, const double *u, const double *v, double *out)
Definition: functions.h:201
double TanhTable[kTableSize]
Definition: functions.cpp:23
double operator()(double y) const
Definition: functions.h:124
const double kScaleFactor
Definition: functions.h:39
void FuncMultiply(const double *u, const double *v, int n, double *out)
Definition: functions.h:155
double operator()(double y) const
Definition: functions.h:132
double operator()(double x) const
Definition: functions.h:102
double operator()(double y) const
Definition: functions.h:108
double Tanh(double x)
Definition: functions.h:45
void SumVectors(int n, const double *v1, const double *v2, const double *v3, const double *v4, const double *v5, double *sum)
Definition: functions.h:209
double operator()(double x) const
Definition: functions.h:111
void AccumulateVector(int n, const double *src, double *dest)
Definition: functions.h:191
void ZeroVector(int n, T *vec)
Definition: functions.h:219
double Logistic(double x)
Definition: functions.h:63
T ClipToRange(const T &x, const T &lower_bound, const T &upper_bound)
Definition: helpers.h:111
double operator()(double x) const
Definition: functions.h:117
double operator()(double x) const
Definition: functions.h:129
double operator()(double x) const
Definition: functions.h:141
const int kTableSize
Definition: functions.h:37