Debian Codegraph @

src/nsUniversalDetector.cpp

raw
1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* ***** BEGIN LICENSE BLOCK *****
3 * Version: MPL 1.1/GPL 2.0/LGPL 2.1
4 *
5 * The contents of this file are subject to the Mozilla Public License Version
6 * 1.1 (the "License"); you may not use this file except in compliance with
7 * the License. You may obtain a copy of the License at
8 * http://www.mozilla.org/MPL/
9 *
10 * Software distributed under the License is distributed on an "AS IS" basis,
11 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
12 * for the specific language governing rights and limitations under the
13 * License.
14 *
15 * The Original Code is Mozilla Universal charset detector code.
16 *
17 * The Initial Developer of the Original Code is
18 * Netscape Communications Corporation.
19 * Portions created by the Initial Developer are Copyright (C) 2001
20 * the Initial Developer. All Rights Reserved.
21 *
22 * Contributor(s):
23 * Shy Shalom <[email protected]>
24 *
25 * Alternatively, the contents of this file may be used under the terms of
26 * either the GNU General Public License Version 2 or later (the "GPL"), or
27 * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
28 * in which case the provisions of the GPL or the LGPL are applicable instead
29 * of those above. If you wish to allow use of your version of this file only
30 * under the terms of either the GPL or the LGPL, and not to allow others to
31 * use your version of this file under the terms of the MPL, indicate your
32 * decision by deleting the provisions above and replace them with the notice
33 * and other provisions required by the GPL or the LGPL. If you do not delete
34 * the provisions above, a recipient may use your version of this file under
35 * the terms of any one of the MPL, the GPL or the LGPL.
36 *
37 * ***** END LICENSE BLOCK ***** */
38
39 #include "nscore.h"
40
41 #include "nsUniversalDetector.h"
42
43 #include "nsMBCSGroupProber.h"
44 #include "nsSBCSGroupProber.h"
45 #include "nsEscCharsetProber.h"
46 #include "nsLatin1Prober.h"
47
48 nsUniversalDetector::nsUniversalDetector(PRUint32 aLanguageFilter)
49 {
50 mNbspFound = PR_FALSE;
51 mDone = PR_FALSE;
52 mBestGuess = -1; //illegal value as signal
53 mInTag = PR_FALSE;
54 mEscCharSetProber = nsnull;
55
56 mStart = PR_TRUE;
57 mDetectedCharset = nsnull;
58 mGotData = PR_FALSE;
59 mInputState = ePureAscii;
60 mLastChar = '\0';
61 mLanguageFilter = aLanguageFilter;
62
63 PRUint32 i;
64 for (i = 0; i < NUM_OF_CHARSET_PROBERS; i++)
65 mCharSetProbers[i] = nsnull;
66 }
67
68 nsUniversalDetector::~nsUniversalDetector()
69 {
70 for (PRInt32 i = 0; i < NUM_OF_CHARSET_PROBERS; i++)
71 delete mCharSetProbers[i];
72
73 delete mEscCharSetProber;
74 }
75
76 void
77 nsUniversalDetector::Reset()
78 {
79 mNbspFound = PR_FALSE;
80 mDone = PR_FALSE;
81 mBestGuess = -1; //illegal value as signal
82 mInTag = PR_FALSE;
83
84 mStart = PR_TRUE;
85 mDetectedCharset = nsnull;
86 mGotData = PR_FALSE;
87 mInputState = ePureAscii;
88 mLastChar = '\0';
89
90 if (mEscCharSetProber)
91 mEscCharSetProber->Reset();
92
93 PRUint32 i;
94 for (i = 0; i < NUM_OF_CHARSET_PROBERS; i++)
95 if (mCharSetProbers[i])
96 mCharSetProbers[i]->Reset();
97 }
98
99 //---------------------------------------------------------------------
100 #define SHORTCUT_THRESHOLD (float)0.95
101 #define MINIMUM_THRESHOLD (float)0.20
102
103 nsresult nsUniversalDetector::HandleData(const char* aBuf, PRUint32 aLen)
104 {
105 if(mDone)
106 return NS_OK;
107
108 if (aLen > 0)
109 mGotData = PR_TRUE;
110
111 /* If the data starts with BOM, we know it is UTF. */
112 if (mStart)
113 {
114 mStart = PR_FALSE;
115 if (aLen > 2)
116 {
117 switch (aBuf[0])
118 {
119 case '\xEF':
120 if (('\xBB' == aBuf[1]) && ('\xBF' == aBuf[2]))
121 /* EF BB BF: UTF-8 encoded BOM. */
122 mDetectedCharset = "UTF-8";
123 break;
124 case '\xFE':
125 if ('\xFF' == aBuf[1])
126 /* FE FF: UTF-16, big endian BOM. */
127 mDetectedCharset = "UTF-16";
128 break;
129 case '\xFF':
130 if ('\xFE' == aBuf[1])
131 {
132 if (aLen > 3 &&
133 aBuf[2] == '\x00' &&
134 aBuf[3] == '\x00')
135 {
136 /* FF FE 00 00: UTF-32 (LE). */
137 mDetectedCharset = "UTF-32";
138 }
139 else
140 {
141 /* FF FE: UTF-16, little endian BOM. */
142 mDetectedCharset = "UTF-16";
143 }
144 }
145 break;
146 case '\x00':
147 if (aLen > 3 &&
148 aBuf[1] == '\x00' &&
149 aBuf[2] == '\xFE' &&
150 aBuf[3] == '\xFF')
151 {
152 /* 00 00 FE FF: UTF-32 (BE). */
153 mDetectedCharset = "UTF-32";
154 }
155 break;
156 }
157 }
158
159 if (mDetectedCharset)
160 {
161 mDone = PR_TRUE;
162 return NS_OK;
163 }
164 }
165
166 PRUint32 i;
167 for (i = 0; i < aLen; i++)
168 {
169 /* If every other character is ASCII or 0xA0, we don't run charset
170 * probers.
171 * 0xA0 (NBSP in a few charset) is apparently a rare exception
172 * of non-ASCII character often contained in nearly-ASCII text. */
173 if (aBuf[i] & '\x80' && aBuf[i] != '\xA0')
174 {
175 /* We got a non-ASCII byte (high-byte) */
176 if (mInputState != eHighbyte)
177 {
178 //adjust state
179 mInputState = eHighbyte;
180
181 //kill mEscCharSetProber if it is active
182 if (mEscCharSetProber) {
183 delete mEscCharSetProber;
184 mEscCharSetProber = nsnull;
185 }
186
187 //start multibyte and singlebyte charset prober
188 if (nsnull == mCharSetProbers[0])
189 {
190 mCharSetProbers[0] = new nsMBCSGroupProber(mLanguageFilter);
191 if (nsnull == mCharSetProbers[0])
192 return NS_ERROR_OUT_OF_MEMORY;
193 }
194 if (nsnull == mCharSetProbers[1] &&
195 (mLanguageFilter & NS_FILTER_NON_CJK))
196 {
197 mCharSetProbers[1] = new nsSBCSGroupProber;
198 if (nsnull == mCharSetProbers[1])
199 return NS_ERROR_OUT_OF_MEMORY;
200 }
201 if (nsnull == mCharSetProbers[2])
202 {
203 mCharSetProbers[2] = new nsLatin1Prober;
204 if (nsnull == mCharSetProbers[2])
205 return NS_ERROR_OUT_OF_MEMORY;
206 }
207 }
208 }
209 else
210 {
211 /* Just pure ASCII or NBSP so far. */
212 if (aBuf[i] == '\xA0')
213 {
214 /* ASCII with the only exception of NBSP seems quite common.
215 * I doubt it is really necessary to train a model here, so let's
216 * just make an exception.
217 */
218 mNbspFound = PR_TRUE;
219 }
220 else if (mInputState == ePureAscii &&
221 (aBuf[i] == '\033' || (aBuf[i] == '{' && mLastChar == '~')))
222 {
223 /* We found an escape character or HZ "~{". */
224 mInputState = eEscAscii;
225 }
226 mLastChar = aBuf[i];
227 }
228 }
229
230 nsProbingState st;
231 switch (mInputState)
232 {
233 case eEscAscii:
234 if (nsnull == mEscCharSetProber) {
235 mEscCharSetProber = new nsEscCharSetProber(mLanguageFilter);
236 if (nsnull == mEscCharSetProber)
237 return NS_ERROR_OUT_OF_MEMORY;
238 }
239 st = mEscCharSetProber->HandleData(aBuf, aLen);
240 if (st == eFoundIt)
241 {
242 mDone = PR_TRUE;
243 mDetectedCharset = mEscCharSetProber->GetCharSetName();
244 }
245 break;
246 case eHighbyte:
247 for (i = 0; i < NUM_OF_CHARSET_PROBERS; i++)
248 {
249 if (mCharSetProbers[i])
250 {
251 st = mCharSetProbers[i]->HandleData(aBuf, aLen);
252 if (st == eFoundIt)
253 {
254 mDone = PR_TRUE;
255 mDetectedCharset = mCharSetProbers[i]->GetCharSetName();
256 return NS_OK;
257 }
258 }
259 }
260 break;
261
262 default:
263 break;
264 }
265 return NS_OK;
266 }
267
268
269 //---------------------------------------------------------------------
270 void nsUniversalDetector::DataEnd()
271 {
272 if (!mGotData)
273 {
274 // we haven't got any data yet, return immediately
275 // caller program sometimes call DataEnd before anything has been sent to detector
276 return;
277 }
278
279 if (! mDetectedCharset)
280 {
281 switch (mInputState)
282 {
283 case eEscAscii:
284 case ePureAscii:
285 if (mNbspFound)
286 {
287 /* ISO-8859-1 is a good result candidate for ASCII + NBSP.
288 * (though it could have been any ISO-8859 encoding). */
289 mDetectedCharset = "ISO-8859-1";
290 }
291 else
292 {
293 /* ASCII with the ESC character (or the sequence "~{") is still
294 * ASCII until proven otherwise. */
295 mDetectedCharset = "ASCII";
296 }
297 default:
298 break;
299 }
300 }
301
302 if (mDetectedCharset)
303 {
304 mDone = PR_TRUE;
305 Report(mDetectedCharset);
306 return;
307 }
308
309 switch (mInputState)
310 {
311 case eHighbyte:
312 {
313 float proberConfidence;
314 float maxProberConfidence = (float)0.0;
315 PRInt32 maxProber = 0;
316
317 for (PRInt32 i = 0; i < NUM_OF_CHARSET_PROBERS; i++)
318 {
319 if (mCharSetProbers[i])
320 {
321 proberConfidence = mCharSetProbers[i]->GetConfidence();
322 if (proberConfidence > maxProberConfidence)
323 {
324 maxProberConfidence = proberConfidence;
325 maxProber = i;
326 }
327 }
328 }
329 //do not report anything because we are not confident of it, that's in fact a negative answer
330 if (maxProberConfidence > MINIMUM_THRESHOLD)
331 Report(mCharSetProbers[maxProber]->GetCharSetName());
332 }
333 break;
334 case eEscAscii:
335 break;
336 default:
337 ;
338 }
339 return;
340 }