Debian Codegraph @

src/nsSBCharSetProber.h

raw
1 /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* ***** BEGIN LICENSE BLOCK *****
3 * Version: MPL 1.1/GPL 2.0/LGPL 2.1
4 *
5 * The contents of this file are subject to the Mozilla Public License Version
6 * 1.1 (the "License"); you may not use this file except in compliance with
7 * the License. You may obtain a copy of the License at
8 * http://www.mozilla.org/MPL/
9 *
10 * Software distributed under the License is distributed on an "AS IS" basis,
11 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
12 * for the specific language governing rights and limitations under the
13 * License.
14 *
15 * The Original Code is Mozilla Universal charset detector code.
16 *
17 * The Initial Developer of the Original Code is
18 * Netscape Communications Corporation.
19 * Portions created by the Initial Developer are Copyright (C) 2001
20 * the Initial Developer. All Rights Reserved.
21 *
22 * Contributor(s):
23 * Shy Shalom <[email protected]>
24 *
25 * Alternatively, the contents of this file may be used under the terms of
26 * either the GNU General Public License Version 2 or later (the "GPL"), or
27 * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
28 * in which case the provisions of the GPL or the LGPL are applicable instead
29 * of those above. If you wish to allow use of your version of this file only
30 * under the terms of either the GPL or the LGPL, and not to allow others to
31 * use your version of this file under the terms of the MPL, indicate your
32 * decision by deleting the provisions above and replace them with the notice
33 * and other provisions required by the GPL or the LGPL. If you do not delete
34 * the provisions above, a recipient may use your version of this file under
35 * the terms of any one of the MPL, the GPL or the LGPL.
36 *
37 * ***** END LICENSE BLOCK ***** */
38 #ifndef nsSingleByteCharSetProber_h__
39 #define nsSingleByteCharSetProber_h__
40
41 #include "nsCharSetProber.h"
42
43 /** Codepoints **/
44
45 /* Illegal codepoints.*/
46 #define ILL 255
47 /* Control character. */
48 #define CTR 254
49 /* Symbols and punctuation that does not belong to words. */
50 #define SYM 253
51 /* Return/Line feeds. */
52 #define RET 252
53 /* Numbers 0-9. */
54 #define NUM 251
55
56 #define SB_ENOUGH_REL_THRESHOLD 1024
57 #define POSITIVE_SHORTCUT_THRESHOLD (float)0.95
58 #define NEGATIVE_SHORTCUT_THRESHOLD (float)0.05
59 #define SYMBOL_CAT_ORDER 250
60
61 #define NUMBER_OF_SEQ_CAT 4
62 #define POSITIVE_CAT (NUMBER_OF_SEQ_CAT-1)
63 #define PROBABLE_CAT (NUMBER_OF_SEQ_CAT-2)
64 #define NEUTRAL_CAT (NUMBER_OF_SEQ_CAT-3)
65 #define NEGATIVE_CAT 0
66
67 typedef struct
68 {
69 /* [256] table mapping codepoints to chararacter orders. */
70 const unsigned char* const charToOrderMap;
71 /* freqCharCount x freqCharCount table of 2-char sequence's frequencies. */
72 const PRUint8* const precedenceMatrix;
73 /* The count of frequent characters. */
74 int freqCharCount;
75 float mTypicalPositiveRatio; // = freqSeqs / totalSeqs
76 PRBool keepEnglishLetter; // says if this script contains English characters (not implemented)
77 const char* const charsetName;
78 } SequenceModel;
79
80
81 class nsSingleByteCharSetProber : public nsCharSetProber{
82 public:
83 nsSingleByteCharSetProber(const SequenceModel *model)
84 :mModel(model), mReversed(PR_FALSE), mNameProber(0) { Reset(); }
85 nsSingleByteCharSetProber(const SequenceModel *model, PRBool reversed, nsCharSetProber* nameProber)
86 :mModel(model), mReversed(reversed), mNameProber(nameProber) { Reset(); }
87
88 virtual const char* GetCharSetName();
89 virtual nsProbingState HandleData(const char* aBuf, PRUint32 aLen);
90 virtual nsProbingState GetState(void) {return mState;}
91 virtual void Reset(void);
92 virtual float GetConfidence(void);
93 virtual void SetOpion() {}
94
95 // This feature is not implemented yet. any current language model
96 // contain this parameter as PR_FALSE. No one is looking at this
97 // parameter or calling this method.
98 // Moreover, the nsSBCSGroupProber which calls the HandleData of this
99 // prober has a hard-coded call to FilterWithoutEnglishLetters which gets rid
100 // of the English letters.
101 PRBool KeepEnglishLetters() {return mModel->keepEnglishLetter;} // (not implemented)
102
103 #ifdef DEBUG_chardet
104 virtual void DumpStatus();
105 #endif
106
107 protected:
108 nsProbingState mState;
109 const SequenceModel* const mModel;
110 const PRBool mReversed; // PR_TRUE if we need to reverse every pair in the model lookup
111
112 //char order of last character
113 unsigned char mLastOrder;
114
115 PRUint32 mTotalSeqs;
116 PRUint32 mSeqCounters[NUMBER_OF_SEQ_CAT];
117
118 PRUint32 mTotalChar;
119 PRUint32 mCtrlChar;
120 //characters that fall in our sampling range
121 PRUint32 mFreqChar;
122
123 // Optional auxiliary prober for name decision. created and destroyed by the GroupProber
124 nsCharSetProber* mNameProber;
125
126 };
127
128 extern const SequenceModel Windows_1256ArabicModel;
129 extern const SequenceModel Iso_8859_6ArabicModel;
130
131 extern const SequenceModel Koi8rRussianModel;
132 extern const SequenceModel Win1251RussianModel;
133 extern const SequenceModel Latin5RussianModel;
134 extern const SequenceModel MacCyrillicRussianModel;
135 extern const SequenceModel Ibm866RussianModel;
136 extern const SequenceModel Ibm855RussianModel;
137
138 extern const SequenceModel Iso_8859_7GreekModel;
139 extern const SequenceModel Windows_1253GreekModel;
140
141 extern const SequenceModel Latin5BulgarianModel;
142 extern const SequenceModel Win1251BulgarianModel;
143
144 extern const SequenceModel Iso_8859_2HungarianModel;
145 extern const SequenceModel Windows_1250HungarianModel;
146
147 extern const SequenceModel Win1255Model;
148
149 extern const SequenceModel Tis_620ThaiModel;
150 extern const SequenceModel Iso_8859_11ThaiModel;
151
152 extern const SequenceModel Iso_8859_15FrenchModel;
153 extern const SequenceModel Iso_8859_1FrenchModel;
154 extern const SequenceModel Windows_1252FrenchModel;
155
156 extern const SequenceModel Iso_8859_15SpanishModel;
157 extern const SequenceModel Iso_8859_1SpanishModel;
158 extern const SequenceModel Windows_1252SpanishModel;
159
160 extern const SequenceModel Iso_8859_1GermanModel;
161 extern const SequenceModel Windows_1252GermanModel;
162
163 extern const SequenceModel Iso_8859_3EsperantoModel;
164
165 extern const SequenceModel Iso_8859_3TurkishModel;
166 extern const SequenceModel Iso_8859_9TurkishModel;
167
168 extern const SequenceModel VisciiVietnameseModel;
169 extern const SequenceModel Windows_1258VietnameseModel;
170
171 extern const SequenceModel Iso_8859_15DanishModel;
172 extern const SequenceModel Iso_8859_1DanishModel;
173 extern const SequenceModel Windows_1252DanishModel;
174 extern const SequenceModel Ibm865DanishModel;
175
176 extern const SequenceModel Iso_8859_13LithuanianModel;
177 extern const SequenceModel Iso_8859_10LithuanianModel;
178 extern const SequenceModel Iso_8859_4LithuanianModel;
179
180 extern const SequenceModel Iso_8859_13LatvianModel;
181 extern const SequenceModel Iso_8859_10LatvianModel;
182 extern const SequenceModel Iso_8859_4LatvianModel;
183
184 extern const SequenceModel Iso_8859_1PortugueseModel;
185 extern const SequenceModel Iso_8859_9PortugueseModel;
186 extern const SequenceModel Iso_8859_15PortugueseModel;
187 extern const SequenceModel Windows_1252PortugueseModel;
188
189 extern const SequenceModel Iso_8859_3MalteseModel;
190
191 extern const SequenceModel Windows_1250CzechModel;
192 extern const SequenceModel Iso_8859_2CzechModel;
193 extern const SequenceModel Ibm852CzechModel;
194 extern const SequenceModel Mac_CentraleuropeCzechModel;
195
196 extern const SequenceModel Windows_1250SlovakModel;
197 extern const SequenceModel Iso_8859_2SlovakModel;
198 extern const SequenceModel Ibm852SlovakModel;
199 extern const SequenceModel Mac_CentraleuropeSlovakModel;
200
201 extern const SequenceModel Windows_1250PolishModel;
202 extern const SequenceModel Iso_8859_2PolishModel;
203 extern const SequenceModel Iso_8859_13PolishModel;
204 extern const SequenceModel Iso_8859_16PolishModel;
205 extern const SequenceModel Ibm852PolishModel;
206 extern const SequenceModel Mac_CentraleuropePolishModel;
207
208 extern const SequenceModel Iso_8859_1FinnishModel;
209 extern const SequenceModel Iso_8859_4FinnishModel;
210 extern const SequenceModel Iso_8859_9FinnishModel;
211 extern const SequenceModel Iso_8859_13FinnishModel;
212 extern const SequenceModel Iso_8859_15FinnishModel;
213 extern const SequenceModel Windows_1252FinnishModel;
214
215 extern const SequenceModel Iso_8859_1ItalianModel;
216 extern const SequenceModel Iso_8859_3ItalianModel;
217 extern const SequenceModel Iso_8859_9ItalianModel;
218 extern const SequenceModel Iso_8859_15ItalianModel;
219 extern const SequenceModel Windows_1252ItalianModel;
220
221 extern const SequenceModel Windows_1250CroatianModel;
222 extern const SequenceModel Iso_8859_2CroatianModel;
223 extern const SequenceModel Iso_8859_13CroatianModel;
224 extern const SequenceModel Iso_8859_16CroatianModel;
225 extern const SequenceModel Ibm852CroatianModel;
226 extern const SequenceModel Mac_CentraleuropeCroatianModel;
227
228 extern const SequenceModel Windows_1252EstonianModel;
229 extern const SequenceModel Windows_1257EstonianModel;
230 extern const SequenceModel Iso_8859_4EstonianModel;
231 extern const SequenceModel Iso_8859_13EstonianModel;
232 extern const SequenceModel Iso_8859_15EstonianModel;
233
234 extern const SequenceModel Iso_8859_15IrishModel;
235 extern const SequenceModel Iso_8859_9IrishModel;
236 extern const SequenceModel Iso_8859_1IrishModel;
237 extern const SequenceModel Windows_1252IrishModel;
238
239 extern const SequenceModel Windows_1250RomanianModel;
240 extern const SequenceModel Iso_8859_2RomanianModel;
241 extern const SequenceModel Iso_8859_16RomanianModel;
242 extern const SequenceModel Ibm852RomanianModel;
243
244 extern const SequenceModel Windows_1250SloveneModel;
245 extern const SequenceModel Iso_8859_2SloveneModel;
246 extern const SequenceModel Iso_8859_16SloveneModel;
247 extern const SequenceModel Ibm852SloveneModel;
248 extern const SequenceModel Mac_CentraleuropeSloveneModel;
249
250 extern const SequenceModel Iso_8859_1SwedishModel;
251 extern const SequenceModel Iso_8859_4SwedishModel;
252 extern const SequenceModel Iso_8859_9SwedishModel;
253 extern const SequenceModel Iso_8859_15SwedishModel;
254 extern const SequenceModel Windows_1252SwedishModel;
255
256 extern const SequenceModel Iso_8859_15NorwegianModel;
257 extern const SequenceModel Iso_8859_1NorwegianModel;
258 extern const SequenceModel Windows_1252NorwegianModel;
259 extern const SequenceModel Ibm865NorwegianModel;
260
261 #endif /* nsSingleByteCharSetProber_h__ */
262