tesseract  v4.0.0-17-g361f3264
Open Source OCR Engine
ambigs.h
1 // File: ambigs.h
3 // Description: Constants, flags, functions for dealing with
4 // ambiguities (training and recognition).
5 // Author: Daria Antonova
6 // Created: Mon Aug 23 11:26:43 PDT 2008
7 //
8 // (C) Copyright 2008, Google Inc.
9 // Licensed under the Apache License, Version 2.0 (the "License");
10 // you may not use this file except in compliance with the License.
11 // You may obtain a copy of the License at
12 // http://www.apache.org/licenses/LICENSE-2.0
13 // Unless required by applicable law or agreed to in writing, software
14 // distributed under the License is distributed on an "AS IS" BASIS,
15 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 // See the License for the specific language governing permissions and
17 // limitations under the License.
18 //
20 
21 #ifndef TESSERACT_CCUTIL_AMBIGS_H_
22 #define TESSERACT_CCUTIL_AMBIGS_H_
23 
24 #include "elst.h"
25 #include "tprintf.h"
26 #include "unichar.h"
27 #include "unicharset.h"
28 #include "genericvector.h"
29 
30 #define MAX_AMBIG_SIZE 10
31 
32 namespace tesseract {
33 
35 
36 static const int kUnigramAmbigsBufferSize = 1000;
37 static const char kAmbigNgramSeparator[] = { ' ', '\0' };
38 static const char kAmbigDelimiters[] = "\t ";
39 static const char kIllegalMsg[] =
40  "Illegal ambiguity specification on line %d\n";
41 static const char kIllegalUnicharMsg[] =
42  "Illegal unichar %s in ambiguity specification\n";
43 
44 enum AmbigType {
45  NOT_AMBIG, // the ngram pair is not ambiguous
46  REPLACE_AMBIG, // ocred ngram should always be substituted with correct
47  DEFINITE_AMBIG, // add correct ngram to the classifier results (1-1)
48  SIMILAR_AMBIG, // use pairwise classifier for ocred/correct pair (1-1)
49  CASE_AMBIG, // this is a case ambiguity (1-1)
50 
51  AMBIG_TYPE_COUNT // number of enum entries
52 };
53 
54 // A collection of utility functions for arrays of UNICHAR_IDs that are
55 // terminated by INVALID_UNICHAR_ID.
57  public:
58  // Compares two arrays of unichar ids. Returns -1 if the length of array1 is
59  // less than length of array2, if any array1[i] is less than array2[i].
60  // Returns 0 if the arrays are equal, 1 otherwise.
61  // The function assumes that the arrays are terminated by INVALID_UNICHAR_ID.
62  static inline int compare(const UNICHAR_ID *ptr1, const UNICHAR_ID *ptr2) {
63  for (;;) {
64  const UNICHAR_ID val1 = *ptr1++;
65  const UNICHAR_ID val2 = *ptr2++;
66  if (val1 != val2) {
67  if (val1 == INVALID_UNICHAR_ID) return -1;
68  if (val2 == INVALID_UNICHAR_ID) return 1;
69  if (val1 < val2) return -1;
70  return 1;
71  }
72  if (val1 == INVALID_UNICHAR_ID) return 0;
73  }
74  }
75 
76  // Look uid in the vector of uids. If found, the index of the matched
77  // element is returned. Otherwise, it returns -1.
78  static inline int find_in(const UnicharIdVector& uid_vec,
79  const UNICHAR_ID uid) {
80  for (int i = 0; i < uid_vec.size(); ++i)
81  if (uid_vec[i] == uid) return i;
82  return -1;
83  }
84 
85  // Copies UNICHAR_IDs from dst to src. Returns the number of ids copied.
86  // The function assumes that the arrays are terminated by INVALID_UNICHAR_ID
87  // and that dst has enough space for all the elements from src.
88  static inline int copy(const UNICHAR_ID src[], UNICHAR_ID dst[]) {
89  int i = 0;
90  do {
91  dst[i] = src[i];
92  } while (dst[i++] != INVALID_UNICHAR_ID);
93  return i - 1;
94  }
95 
96  // Prints unichars corresponding to the unichar_ids in the given array.
97  // The function assumes that array is terminated by INVALID_UNICHAR_ID.
98  static inline void print(const UNICHAR_ID array[],
99  const UNICHARSET &unicharset) {
100  const UNICHAR_ID *ptr = array;
101  if (*ptr == INVALID_UNICHAR_ID) tprintf("[Empty]");
102  while (*ptr != INVALID_UNICHAR_ID) {
103  tprintf("%s ", unicharset.id_to_unichar(*ptr++));
104  }
105  tprintf("( ");
106  ptr = array;
107  while (*ptr != INVALID_UNICHAR_ID) tprintf("%d ", *ptr++);
108  tprintf(")\n");
109  }
110 };
111 
112 // AMBIG_SPEC_LIST stores a list of dangerous ambigs that
113 // start with the same unichar (e.g. r->t rn->m rr1->m).
114 class AmbigSpec : public ELIST_LINK {
115  public:
116  AmbigSpec();
117  ~AmbigSpec() = default;
118 
119  // Comparator function for sorting AmbigSpec_LISTs. The lists will
120  // be sorted by their wrong_ngram arrays. Example of wrong_ngram vectors
121  // in a a sorted AmbigSpec_LIST: [9 1 3], [9 3 4], [9 8], [9, 8 1].
122  static int compare_ambig_specs(const void *spec1, const void *spec2) {
123  const AmbigSpec *s1 = *static_cast<const AmbigSpec *const *>(spec1);
124  const AmbigSpec *s2 = *static_cast<const AmbigSpec *const *>(spec2);
126  if (result != 0) return result;
128  s2->correct_fragments);
129  }
130 
131  UNICHAR_ID wrong_ngram[MAX_AMBIG_SIZE + 1];
132  UNICHAR_ID correct_fragments[MAX_AMBIG_SIZE + 1];
133  UNICHAR_ID correct_ngram_id;
136 };
137 ELISTIZEH(AmbigSpec)
138 
139 // AMBIG_TABLE[i] stores a set of ambiguities whose
140 // wrong ngram starts with unichar id i.
142 
144  public:
145  UnicharAmbigs() = default;
147  replace_ambigs_.delete_data_pointers();
148  dang_ambigs_.delete_data_pointers();
149  one_to_one_definite_ambigs_.delete_data_pointers();
150  }
151 
152  const UnicharAmbigsVector &dang_ambigs() const { return dang_ambigs_; }
153  const UnicharAmbigsVector &replace_ambigs() const { return replace_ambigs_; }
154 
155  // Initializes the ambigs by adding a nullptr pointer to each table.
156  void InitUnicharAmbigs(const UNICHARSET& unicharset,
157  bool use_ambigs_for_adaption);
158 
159  // Loads the universal ambigs that are useful for any language.
160  void LoadUniversal(const UNICHARSET& encoder_set, UNICHARSET* unicharset);
161 
162  // Fills in two ambiguity tables (replaceable and dangerous) with information
163  // read from the ambigs file. An ambiguity table is an array of lists.
164  // The array is indexed by a class id. Each entry in the table provides
165  // a list of potential ambiguities which can start with the corresponding
166  // character. For example the ambiguity "rn -> m", would be located in the
167  // table at index of unicharset.unichar_to_id('r').
168  // In 1-1 ambiguities (e.g. s -> S, 1 -> I) are recorded in
169  // one_to_one_definite_ambigs_. This vector is also indexed by the class id
170  // of the wrong part of the ambiguity and each entry contains a vector of
171  // unichar ids that are ambiguous to it.
172  // encoder_set is used to encode the ambiguity strings, undisturbed by new
173  // unichar_ids that may be created by adding the ambigs.
174  void LoadUnicharAmbigs(const UNICHARSET& encoder_set,
175  TFile *ambigs_file, int debug_level,
176  bool use_ambigs_for_adaption, UNICHARSET *unicharset);
177 
178  // Returns definite 1-1 ambigs for the given unichar id.
180  UNICHAR_ID unichar_id) const {
181  if (one_to_one_definite_ambigs_.empty()) return nullptr;
182  return one_to_one_definite_ambigs_[unichar_id];
183  }
184 
185  // Returns a pointer to the vector with all unichar ids that appear in the
186  // 'correct' part of the ambiguity pair when the given unichar id appears
187  // in the 'wrong' part of the ambiguity. E.g. if DangAmbigs file consist of
188  // m->rn,rn->m,m->iii, UnicharAmbigsForAdaption() called with unichar id of
189  // m will return a pointer to a vector with unichar ids of r,n,i.
191  UNICHAR_ID unichar_id) const {
192  if (ambigs_for_adaption_.empty()) return nullptr;
193  return ambigs_for_adaption_[unichar_id];
194  }
195 
196  // Similar to the above, but return the vector of unichar ids for which
197  // the given unichar_id is an ambiguity (appears in the 'wrong' part of
198  // some ambiguity pair).
200  UNICHAR_ID unichar_id) const {
201  if (reverse_ambigs_for_adaption_.empty()) return nullptr;
202  return reverse_ambigs_for_adaption_[unichar_id];
203  }
204 
205  private:
206  bool ParseAmbiguityLine(int line_num, int version, int debug_level,
207  const UNICHARSET &unicharset, char *buffer,
208  int *test_ambig_part_size,
209  UNICHAR_ID *test_unichar_ids,
210  int *replacement_ambig_part_size,
211  char *replacement_string, int *type);
212  bool InsertIntoTable(UnicharAmbigsVector &table,
213  int test_ambig_part_size, UNICHAR_ID *test_unichar_ids,
214  int replacement_ambig_part_size,
215  const char *replacement_string, int type,
216  AmbigSpec *ambig_spec, UNICHARSET *unicharset);
217 
223 };
224 
225 } // namespace tesseract
226 
227 #endif // TESSERACT_CCUTIL_AMBIGS_H_
static int find_in(const UnicharIdVector &uid_vec, const UNICHAR_ID uid)
Definition: ambigs.h:78
UNICHAR_ID correct_fragments[10+1]
Definition: ambigs.h:132
static const char kAmbigDelimiters[]
Definition: ambigs.h:38
int wrong_ngram_size
Definition: ambigs.h:135
UnicharAmbigsVector replace_ambigs_
Definition: ambigs.h:219
AmbigType
Definition: ambigs.h:44
AmbigType type
Definition: ambigs.h:134
GenericVector< UnicharIdVector * > one_to_one_definite_ambigs_
Definition: ambigs.h:220
static int compare_ambig_specs(const void *spec1, const void *spec2)
Definition: ambigs.h:122
Definition: ambigs.h:45
Definition: unicharset.h:146
Definition: ambigs.h:114
static int compare(const UNICHAR_ID *ptr1, const UNICHAR_ID *ptr2)
Definition: ambigs.h:62
static int copy(const UNICHAR_ID src[], UNICHAR_ID dst[])
Definition: ambigs.h:88
Definition: serialis.h:77
UnicharAmbigsVector dang_ambigs_
Definition: ambigs.h:218
Definition: baseapi.cpp:94
static const char kAmbigNgramSeparator[]
Definition: ambigs.h:37
Definition: ambigs.h:47
static const int kUnigramAmbigsBufferSize
Definition: ambigs.h:36
Definition: ambigs.h:56
Definition: ambigs.h:49
~UnicharAmbigs()
Definition: ambigs.h:146
const UnicharIdVector * ReverseAmbigsForAdaption(UNICHAR_ID unichar_id) const
Definition: ambigs.h:199
Definition: ambigs.h:137
const UnicharIdVector * OneToOneDefiniteAmbigs(UNICHAR_ID unichar_id) const
Definition: ambigs.h:179
static void print(const UNICHAR_ID array[], const UNICHARSET &unicharset)
Definition: ambigs.h:98
Definition: ambigs.h:143
const UnicharIdVector * AmbigsForAdaption(UNICHAR_ID unichar_id) const
Definition: ambigs.h:190
const UnicharAmbigsVector & replace_ambigs() const
Definition: ambigs.h:153
Definition: ambigs.h:46
GenericVector< UnicharIdVector * > reverse_ambigs_for_adaption_
Definition: ambigs.h:222
int size() const
Definition: genericvector.h:71
UNICHAR_ID wrong_ngram[10+1]
Definition: ambigs.h:131
Definition: ambigs.h:48
Definition: ambigs.h:51
const UnicharAmbigsVector & dang_ambigs() const
Definition: ambigs.h:152
GenericVector< UnicharIdVector * > ambigs_for_adaption_
Definition: ambigs.h:221
static const char kIllegalUnicharMsg[]
Definition: ambigs.h:41
static const char kIllegalMsg[]
Definition: ambigs.h:39
const char * id_to_unichar(UNICHAR_ID id) const
Definition: unicharset.cpp:290
UNICHAR_ID correct_ngram_id
Definition: ambigs.h:133