Debian Codegraph @

src/nsSBCSGroupProber.cpp

raw
1 /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* ***** BEGIN LICENSE BLOCK *****
3 * Version: MPL 1.1/GPL 2.0/LGPL 2.1
4 *
5 * The contents of this file are subject to the Mozilla Public License Version
6 * 1.1 (the "License"); you may not use this file except in compliance with
7 * the License. You may obtain a copy of the License at
8 * http://www.mozilla.org/MPL/
9 *
10 * Software distributed under the License is distributed on an "AS IS" basis,
11 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
12 * for the specific language governing rights and limitations under the
13 * License.
14 *
15 * The Original Code is Mozilla Universal charset detector code.
16 *
17 * The Initial Developer of the Original Code is
18 * Netscape Communications Corporation.
19 * Portions created by the Initial Developer are Copyright (C) 2001
20 * the Initial Developer. All Rights Reserved.
21 *
22 * Contributor(s):
23 * Shy Shalom <[email protected]>
24 *
25 * Alternatively, the contents of this file may be used under the terms of
26 * either the GNU General Public License Version 2 or later (the "GPL"), or
27 * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
28 * in which case the provisions of the GPL or the LGPL are applicable instead
29 * of those above. If you wish to allow use of your version of this file only
30 * under the terms of either the GPL or the LGPL, and not to allow others to
31 * use your version of this file under the terms of the MPL, indicate your
32 * decision by deleting the provisions above and replace them with the notice
33 * and other provisions required by the GPL or the LGPL. If you do not delete
34 * the provisions above, a recipient may use your version of this file under
35 * the terms of any one of the MPL, the GPL or the LGPL.
36 *
37 * ***** END LICENSE BLOCK ***** */
38
39 #include <stdio.h>
40 #include "prmem.h"
41
42 #include "nsSBCharSetProber.h"
43 #include "nsSBCSGroupProber.h"
44
45 #include "nsHebrewProber.h"
46
47 nsSBCSGroupProber::nsSBCSGroupProber()
48 {
49 mProbers[0] = new nsSingleByteCharSetProber(&Win1251RussianModel);
50 mProbers[1] = new nsSingleByteCharSetProber(&Koi8rRussianModel);
51 mProbers[2] = new nsSingleByteCharSetProber(&Latin5RussianModel);
52 mProbers[3] = new nsSingleByteCharSetProber(&MacCyrillicRussianModel);
53 mProbers[4] = new nsSingleByteCharSetProber(&Ibm866RussianModel);
54 mProbers[5] = new nsSingleByteCharSetProber(&Ibm855RussianModel);
55
56 mProbers[6] = new nsSingleByteCharSetProber(&Iso_8859_7GreekModel);
57 mProbers[7] = new nsSingleByteCharSetProber(&Windows_1253GreekModel);
58
59 mProbers[8] = new nsSingleByteCharSetProber(&Latin5BulgarianModel);
60 mProbers[9] = new nsSingleByteCharSetProber(&Win1251BulgarianModel);
61
62 nsHebrewProber *hebprober = new nsHebrewProber();
63 // Notice: Any change in these indexes - 10,11,12 must be reflected
64 // in the code below as well.
65 mProbers[10] = hebprober;
66 mProbers[11] = new nsSingleByteCharSetProber(&Win1255Model, PR_FALSE, hebprober); // Logical Hebrew
67 mProbers[12] = new nsSingleByteCharSetProber(&Win1255Model, PR_TRUE, hebprober); // Visual Hebrew
68 // Tell the Hebrew prober about the logical and visual probers
69 if (mProbers[10] && mProbers[11] && mProbers[12]) // all are not null
70 {
71 hebprober->SetModelProbers(mProbers[11], mProbers[12]);
72 }
73 else // One or more is null. avoid any Hebrew probing, null them all
74 {
75 for (PRUint32 i = 10; i <= 12; ++i)
76 {
77 delete mProbers[i];
78 mProbers[i] = 0;
79 }
80 }
81
82 mProbers[13] = new nsSingleByteCharSetProber(&Tis_620ThaiModel);
83 mProbers[14] = new nsSingleByteCharSetProber(&Iso_8859_11ThaiModel);
84
85 mProbers[15] = new nsSingleByteCharSetProber(&Iso_8859_1FrenchModel);
86 mProbers[16] = new nsSingleByteCharSetProber(&Iso_8859_15FrenchModel);
87 mProbers[17] = new nsSingleByteCharSetProber(&Windows_1252FrenchModel);
88
89 mProbers[18] = new nsSingleByteCharSetProber(&Iso_8859_1SpanishModel);
90 mProbers[19] = new nsSingleByteCharSetProber(&Iso_8859_15SpanishModel);
91 mProbers[20] = new nsSingleByteCharSetProber(&Windows_1252SpanishModel);
92
93 mProbers[21] = new nsSingleByteCharSetProber(&Iso_8859_2HungarianModel);
94 mProbers[22] = new nsSingleByteCharSetProber(&Windows_1250HungarianModel);
95
96 mProbers[23] = new nsSingleByteCharSetProber(&Iso_8859_1GermanModel);
97 mProbers[24] = new nsSingleByteCharSetProber(&Windows_1252GermanModel);
98
99 mProbers[25] = new nsSingleByteCharSetProber(&Iso_8859_3EsperantoModel);
100
101 mProbers[26] = new nsSingleByteCharSetProber(&Iso_8859_3TurkishModel);
102 mProbers[27] = new nsSingleByteCharSetProber(&Iso_8859_9TurkishModel);
103
104 mProbers[28] = new nsSingleByteCharSetProber(&Iso_8859_6ArabicModel);
105 mProbers[29] = new nsSingleByteCharSetProber(&Windows_1256ArabicModel);
106
107 mProbers[30] = new nsSingleByteCharSetProber(&VisciiVietnameseModel);
108 mProbers[31] = new nsSingleByteCharSetProber(&Windows_1258VietnameseModel);
109
110 mProbers[32] = new nsSingleByteCharSetProber(&Iso_8859_15DanishModel);
111 mProbers[33] = new nsSingleByteCharSetProber(&Iso_8859_1DanishModel);
112 mProbers[34] = new nsSingleByteCharSetProber(&Windows_1252DanishModel);
113 mProbers[35] = new nsSingleByteCharSetProber(&Ibm865DanishModel);
114
115 mProbers[36] = new nsSingleByteCharSetProber(&Iso_8859_13LithuanianModel);
116 mProbers[37] = new nsSingleByteCharSetProber(&Iso_8859_10LithuanianModel);
117 mProbers[38] = new nsSingleByteCharSetProber(&Iso_8859_4LithuanianModel);
118
119 mProbers[39] = new nsSingleByteCharSetProber(&Iso_8859_13LatvianModel);
120 mProbers[40] = new nsSingleByteCharSetProber(&Iso_8859_10LatvianModel);
121 mProbers[41] = new nsSingleByteCharSetProber(&Iso_8859_4LatvianModel);
122
123 mProbers[42] = new nsSingleByteCharSetProber(&Iso_8859_1PortugueseModel);
124 mProbers[43] = new nsSingleByteCharSetProber(&Iso_8859_9PortugueseModel);
125 mProbers[44] = new nsSingleByteCharSetProber(&Iso_8859_15PortugueseModel);
126 mProbers[45] = new nsSingleByteCharSetProber(&Windows_1252PortugueseModel);
127
128 mProbers[46] = new nsSingleByteCharSetProber(&Iso_8859_3MalteseModel);
129
130 mProbers[47] = new nsSingleByteCharSetProber(&Windows_1250CzechModel);
131 mProbers[48] = new nsSingleByteCharSetProber(&Iso_8859_2CzechModel);
132 mProbers[49] = new nsSingleByteCharSetProber(&Mac_CentraleuropeCzechModel);
133 mProbers[50] = new nsSingleByteCharSetProber(&Ibm852CzechModel);
134
135 mProbers[51] = new nsSingleByteCharSetProber(&Windows_1250SlovakModel);
136 mProbers[52] = new nsSingleByteCharSetProber(&Iso_8859_2SlovakModel);
137 mProbers[53] = new nsSingleByteCharSetProber(&Mac_CentraleuropeSlovakModel);
138 mProbers[54] = new nsSingleByteCharSetProber(&Ibm852SlovakModel);
139
140 mProbers[55] = new nsSingleByteCharSetProber(&Windows_1250PolishModel);
141 mProbers[56] = new nsSingleByteCharSetProber(&Iso_8859_2PolishModel);
142 mProbers[57] = new nsSingleByteCharSetProber(&Iso_8859_13PolishModel);
143 mProbers[58] = new nsSingleByteCharSetProber(&Iso_8859_16PolishModel);
144 mProbers[59] = new nsSingleByteCharSetProber(&Mac_CentraleuropePolishModel);
145 mProbers[60] = new nsSingleByteCharSetProber(&Ibm852PolishModel);
146
147 mProbers[61] = new nsSingleByteCharSetProber(&Iso_8859_1FinnishModel);
148 mProbers[62] = new nsSingleByteCharSetProber(&Iso_8859_4FinnishModel);
149 mProbers[63] = new nsSingleByteCharSetProber(&Iso_8859_9FinnishModel);
150 mProbers[64] = new nsSingleByteCharSetProber(&Iso_8859_13FinnishModel);
151 mProbers[65] = new nsSingleByteCharSetProber(&Iso_8859_15FinnishModel);
152 mProbers[66] = new nsSingleByteCharSetProber(&Windows_1252FinnishModel);
153
154 mProbers[67] = new nsSingleByteCharSetProber(&Iso_8859_1ItalianModel);
155 mProbers[68] = new nsSingleByteCharSetProber(&Iso_8859_3ItalianModel);
156 mProbers[69] = new nsSingleByteCharSetProber(&Iso_8859_9ItalianModel);
157 mProbers[70] = new nsSingleByteCharSetProber(&Iso_8859_15ItalianModel);
158 mProbers[71] = new nsSingleByteCharSetProber(&Windows_1252ItalianModel);
159
160 mProbers[72] = new nsSingleByteCharSetProber(&Windows_1250CroatianModel);
161 mProbers[73] = new nsSingleByteCharSetProber(&Iso_8859_2CroatianModel);
162 mProbers[74] = new nsSingleByteCharSetProber(&Iso_8859_13CroatianModel);
163 mProbers[75] = new nsSingleByteCharSetProber(&Iso_8859_16CroatianModel);
164 mProbers[76] = new nsSingleByteCharSetProber(&Mac_CentraleuropeCroatianModel);
165 mProbers[77] = new nsSingleByteCharSetProber(&Ibm852CroatianModel);
166
167 mProbers[78] = new nsSingleByteCharSetProber(&Windows_1252EstonianModel);
168 mProbers[79] = new nsSingleByteCharSetProber(&Windows_1257EstonianModel);
169 mProbers[80] = new nsSingleByteCharSetProber(&Iso_8859_4EstonianModel);
170 mProbers[81] = new nsSingleByteCharSetProber(&Iso_8859_13EstonianModel);
171 mProbers[82] = new nsSingleByteCharSetProber(&Iso_8859_15EstonianModel);
172
173 mProbers[83] = new nsSingleByteCharSetProber(&Iso_8859_1IrishModel);
174 mProbers[84] = new nsSingleByteCharSetProber(&Iso_8859_9IrishModel);
175 mProbers[85] = new nsSingleByteCharSetProber(&Iso_8859_15IrishModel);
176 mProbers[86] = new nsSingleByteCharSetProber(&Windows_1252IrishModel);
177
178 mProbers[87] = new nsSingleByteCharSetProber(&Windows_1250RomanianModel);
179 mProbers[88] = new nsSingleByteCharSetProber(&Iso_8859_2RomanianModel);
180 mProbers[89] = new nsSingleByteCharSetProber(&Iso_8859_16RomanianModel);
181 mProbers[90] = new nsSingleByteCharSetProber(&Ibm852RomanianModel);
182
183 mProbers[91] = new nsSingleByteCharSetProber(&Windows_1250SloveneModel);
184 mProbers[92] = new nsSingleByteCharSetProber(&Iso_8859_2SloveneModel);
185 mProbers[93] = new nsSingleByteCharSetProber(&Iso_8859_16SloveneModel);
186 mProbers[94] = new nsSingleByteCharSetProber(&Mac_CentraleuropeSloveneModel);
187 mProbers[95] = new nsSingleByteCharSetProber(&Ibm852SloveneModel);
188
189 mProbers[96] = new nsSingleByteCharSetProber(&Iso_8859_1SwedishModel);
190 mProbers[97] = new nsSingleByteCharSetProber(&Iso_8859_4SwedishModel);
191 mProbers[98] = new nsSingleByteCharSetProber(&Iso_8859_9SwedishModel);
192 mProbers[99] = new nsSingleByteCharSetProber(&Iso_8859_15SwedishModel);
193 mProbers[100] = new nsSingleByteCharSetProber(&Windows_1252SwedishModel);
194
195 mProbers[101] = new nsSingleByteCharSetProber(&Iso_8859_15NorwegianModel);
196 mProbers[102] = new nsSingleByteCharSetProber(&Iso_8859_1NorwegianModel);
197 mProbers[103] = new nsSingleByteCharSetProber(&Windows_1252NorwegianModel);
198 mProbers[104] = new nsSingleByteCharSetProber(&Ibm865NorwegianModel);
199
200 Reset();
201 }
202
203 nsSBCSGroupProber::~nsSBCSGroupProber()
204 {
205 for (PRUint32 i = 0; i < NUM_OF_SBCS_PROBERS; i++)
206 {
207 delete mProbers[i];
208 }
209 }
210
211
212 const char* nsSBCSGroupProber::GetCharSetName()
213 {
214 //if we have no answer yet
215 if (mBestGuess == -1)
216 {
217 GetConfidence();
218 //no charset seems positive
219 if (mBestGuess == -1)
220 //we will use default.
221 mBestGuess = 0;
222 }
223 return mProbers[mBestGuess]->GetCharSetName();
224 }
225
226 void nsSBCSGroupProber::Reset(void)
227 {
228 mActiveNum = 0;
229 for (PRUint32 i = 0; i < NUM_OF_SBCS_PROBERS; i++)
230 {
231 if (mProbers[i]) // not null
232 {
233 mProbers[i]->Reset();
234 mIsActive[i] = PR_TRUE;
235 ++mActiveNum;
236 }
237 else
238 mIsActive[i] = PR_FALSE;
239 }
240 mBestGuess = -1;
241 mState = eDetecting;
242 }
243
244
245 nsProbingState nsSBCSGroupProber::HandleData(const char* aBuf, PRUint32 aLen)
246 {
247 nsProbingState st;
248 PRUint32 i;
249 char *newBuf1 = 0;
250 PRUint32 newLen1 = 0;
251
252 //apply filter to original buffer, and we got new buffer back
253 //depend on what script it is, we will feed them the new buffer
254 //we got after applying proper filter
255 //this is done without any consideration to KeepEnglishLetters
256 //of each prober since as of now, there are no probers here which
257 //recognize languages with English characters.
258 if (!FilterWithoutEnglishLetters(aBuf, aLen, &newBuf1, newLen1))
259 goto done;
260
261 if (newLen1 == 0)
262 goto done; // Nothing to see here, move on.
263
264 for (i = 0; i < NUM_OF_SBCS_PROBERS; i++)
265 {
266 if (!mIsActive[i])
267 continue;
268 st = mProbers[i]->HandleData(newBuf1, newLen1);
269 if (st == eFoundIt)
270 {
271 mBestGuess = i;
272 mState = eFoundIt;
273 break;
274 }
275 else if (st == eNotMe)
276 {
277 mIsActive[i] = PR_FALSE;
278 mActiveNum--;
279 if (mActiveNum <= 0)
280 {
281 mState = eNotMe;
282 break;
283 }
284 }
285 }
286
287 done:
288 PR_FREEIF(newBuf1);
289
290 return mState;
291 }
292
293 float nsSBCSGroupProber::GetConfidence(void)
294 {
295 PRUint32 i;
296 float bestConf = 0.0, cf;
297
298 switch (mState)
299 {
300 case eFoundIt:
301 return (float)0.99; //sure yes
302 case eNotMe:
303 return (float)0.01; //sure no
304 default:
305 for (i = 0; i < NUM_OF_SBCS_PROBERS; i++)
306 {
307 if (!mIsActive[i])
308 continue;
309 cf = mProbers[i]->GetConfidence();
310 if (bestConf < cf)
311 {
312 bestConf = cf;
313 mBestGuess = i;
314 }
315 }
316 }
317 return bestConf;
318 }
319
320 #ifdef DEBUG_chardet
321 void nsSBCSGroupProber::DumpStatus()
322 {
323 PRUint32 i;
324 float cf;
325
326 cf = GetConfidence();
327 printf(" SBCS Group Prober --------begin status \r\n");
328 for (i = 0; i < NUM_OF_SBCS_PROBERS; i++)
329 {
330 if (!mIsActive[i])
331 printf(" inactive: [%s] (i.e. confidence is too low).\r\n", mProbers[i]->GetCharSetName());
332 else
333 mProbers[i]->DumpStatus();
334 }
335 printf(" SBCS Group found best match [%s] confidence %f.\r\n",
336 mProbers[mBestGuess]->GetCharSetName(), cf);
337 }
338 #endif