| 1 |
/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ |
| 2 |
/* ***** BEGIN LICENSE BLOCK ***** |
| 3 |
* Version: MPL 1.1/GPL 2.0/LGPL 2.1 |
| 4 |
* |
| 5 |
* The contents of this file are subject to the Mozilla Public License Version |
| 6 |
* 1.1 (the "License"); you may not use this file except in compliance with |
| 7 |
* the License. You may obtain a copy of the License at |
| 8 |
* http://www.mozilla.org/MPL/ |
| 9 |
* |
| 10 |
* Software distributed under the License is distributed on an "AS IS" basis, |
| 11 |
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License |
| 12 |
* for the specific language governing rights and limitations under the |
| 13 |
* License. |
| 14 |
* |
| 15 |
* The Original Code is Mozilla Universal charset detector code. |
| 16 |
* |
| 17 |
* The Initial Developer of the Original Code is |
| 18 |
* Netscape Communications Corporation. |
| 19 |
* Portions created by the Initial Developer are Copyright (C) 2001 |
| 20 |
* the Initial Developer. All Rights Reserved. |
| 21 |
* |
| 22 |
* Contributor(s): |
| 23 |
* Shy Shalom <[email protected]> |
| 24 |
* |
| 25 |
* Alternatively, the contents of this file may be used under the terms of |
| 26 |
* either the GNU General Public License Version 2 or later (the "GPL"), or |
| 27 |
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), |
| 28 |
* in which case the provisions of the GPL or the LGPL are applicable instead |
| 29 |
* of those above. If you wish to allow use of your version of this file only |
| 30 |
* under the terms of either the GPL or the LGPL, and not to allow others to |
| 31 |
* use your version of this file under the terms of the MPL, indicate your |
| 32 |
* decision by deleting the provisions above and replace them with the notice |
| 33 |
* and other provisions required by the GPL or the LGPL. If you do not delete |
| 34 |
* the provisions above, a recipient may use your version of this file under |
| 35 |
* the terms of any one of the MPL, the GPL or the LGPL. |
| 36 |
* |
| 37 |
* ***** END LICENSE BLOCK ***** */ |
| 38 |
|
| 39 |
#include <stdio.h> |
| 40 |
#include "prmem.h" |
| 41 |
|
| 42 |
#include "nsSBCharSetProber.h" |
| 43 |
#include "nsSBCSGroupProber.h" |
| 44 |
|
| 45 |
#include "nsHebrewProber.h" |
| 46 |
|
| 47 |
nsSBCSGroupProber::nsSBCSGroupProber() |
| 48 |
{ |
| 49 |
mProbers[0] = new nsSingleByteCharSetProber(&Win1251RussianModel); |
| 50 |
mProbers[1] = new nsSingleByteCharSetProber(&Koi8rRussianModel); |
| 51 |
mProbers[2] = new nsSingleByteCharSetProber(&Latin5RussianModel); |
| 52 |
mProbers[3] = new nsSingleByteCharSetProber(&MacCyrillicRussianModel); |
| 53 |
mProbers[4] = new nsSingleByteCharSetProber(&Ibm866RussianModel); |
| 54 |
mProbers[5] = new nsSingleByteCharSetProber(&Ibm855RussianModel); |
| 55 |
|
| 56 |
mProbers[6] = new nsSingleByteCharSetProber(&Iso_8859_7GreekModel); |
| 57 |
mProbers[7] = new nsSingleByteCharSetProber(&Windows_1253GreekModel); |
| 58 |
|
| 59 |
mProbers[8] = new nsSingleByteCharSetProber(&Latin5BulgarianModel); |
| 60 |
mProbers[9] = new nsSingleByteCharSetProber(&Win1251BulgarianModel); |
| 61 |
|
| 62 |
nsHebrewProber *hebprober = new nsHebrewProber(); |
| 63 |
// Notice: Any change in these indexes - 10,11,12 must be reflected |
| 64 |
// in the code below as well. |
| 65 |
mProbers[10] = hebprober; |
| 66 |
mProbers[11] = new nsSingleByteCharSetProber(&Win1255Model, PR_FALSE, hebprober); // Logical Hebrew |
| 67 |
mProbers[12] = new nsSingleByteCharSetProber(&Win1255Model, PR_TRUE, hebprober); // Visual Hebrew |
| 68 |
// Tell the Hebrew prober about the logical and visual probers |
| 69 |
if (mProbers[10] && mProbers[11] && mProbers[12]) // all are not null |
| 70 |
{ |
| 71 |
hebprober->SetModelProbers(mProbers[11], mProbers[12]); |
| 72 |
} |
| 73 |
else // One or more is null. avoid any Hebrew probing, null them all |
| 74 |
{ |
| 75 |
for (PRUint32 i = 10; i <= 12; ++i) |
| 76 |
{ |
| 77 |
delete mProbers[i]; |
| 78 |
mProbers[i] = 0; |
| 79 |
} |
| 80 |
} |
| 81 |
|
| 82 |
mProbers[13] = new nsSingleByteCharSetProber(&Tis_620ThaiModel); |
| 83 |
mProbers[14] = new nsSingleByteCharSetProber(&Iso_8859_11ThaiModel); |
| 84 |
|
| 85 |
mProbers[15] = new nsSingleByteCharSetProber(&Iso_8859_1FrenchModel); |
| 86 |
mProbers[16] = new nsSingleByteCharSetProber(&Iso_8859_15FrenchModel); |
| 87 |
mProbers[17] = new nsSingleByteCharSetProber(&Windows_1252FrenchModel); |
| 88 |
|
| 89 |
mProbers[18] = new nsSingleByteCharSetProber(&Iso_8859_1SpanishModel); |
| 90 |
mProbers[19] = new nsSingleByteCharSetProber(&Iso_8859_15SpanishModel); |
| 91 |
mProbers[20] = new nsSingleByteCharSetProber(&Windows_1252SpanishModel); |
| 92 |
|
| 93 |
mProbers[21] = new nsSingleByteCharSetProber(&Iso_8859_2HungarianModel); |
| 94 |
mProbers[22] = new nsSingleByteCharSetProber(&Windows_1250HungarianModel); |
| 95 |
|
| 96 |
mProbers[23] = new nsSingleByteCharSetProber(&Iso_8859_1GermanModel); |
| 97 |
mProbers[24] = new nsSingleByteCharSetProber(&Windows_1252GermanModel); |
| 98 |
|
| 99 |
mProbers[25] = new nsSingleByteCharSetProber(&Iso_8859_3EsperantoModel); |
| 100 |
|
| 101 |
mProbers[26] = new nsSingleByteCharSetProber(&Iso_8859_3TurkishModel); |
| 102 |
mProbers[27] = new nsSingleByteCharSetProber(&Iso_8859_9TurkishModel); |
| 103 |
|
| 104 |
mProbers[28] = new nsSingleByteCharSetProber(&Iso_8859_6ArabicModel); |
| 105 |
mProbers[29] = new nsSingleByteCharSetProber(&Windows_1256ArabicModel); |
| 106 |
|
| 107 |
mProbers[30] = new nsSingleByteCharSetProber(&VisciiVietnameseModel); |
| 108 |
mProbers[31] = new nsSingleByteCharSetProber(&Windows_1258VietnameseModel); |
| 109 |
|
| 110 |
mProbers[32] = new nsSingleByteCharSetProber(&Iso_8859_15DanishModel); |
| 111 |
mProbers[33] = new nsSingleByteCharSetProber(&Iso_8859_1DanishModel); |
| 112 |
mProbers[34] = new nsSingleByteCharSetProber(&Windows_1252DanishModel); |
| 113 |
mProbers[35] = new nsSingleByteCharSetProber(&Ibm865DanishModel); |
| 114 |
|
| 115 |
mProbers[36] = new nsSingleByteCharSetProber(&Iso_8859_13LithuanianModel); |
| 116 |
mProbers[37] = new nsSingleByteCharSetProber(&Iso_8859_10LithuanianModel); |
| 117 |
mProbers[38] = new nsSingleByteCharSetProber(&Iso_8859_4LithuanianModel); |
| 118 |
|
| 119 |
mProbers[39] = new nsSingleByteCharSetProber(&Iso_8859_13LatvianModel); |
| 120 |
mProbers[40] = new nsSingleByteCharSetProber(&Iso_8859_10LatvianModel); |
| 121 |
mProbers[41] = new nsSingleByteCharSetProber(&Iso_8859_4LatvianModel); |
| 122 |
|
| 123 |
mProbers[42] = new nsSingleByteCharSetProber(&Iso_8859_1PortugueseModel); |
| 124 |
mProbers[43] = new nsSingleByteCharSetProber(&Iso_8859_9PortugueseModel); |
| 125 |
mProbers[44] = new nsSingleByteCharSetProber(&Iso_8859_15PortugueseModel); |
| 126 |
mProbers[45] = new nsSingleByteCharSetProber(&Windows_1252PortugueseModel); |
| 127 |
|
| 128 |
mProbers[46] = new nsSingleByteCharSetProber(&Iso_8859_3MalteseModel); |
| 129 |
|
| 130 |
mProbers[47] = new nsSingleByteCharSetProber(&Windows_1250CzechModel); |
| 131 |
mProbers[48] = new nsSingleByteCharSetProber(&Iso_8859_2CzechModel); |
| 132 |
mProbers[49] = new nsSingleByteCharSetProber(&Mac_CentraleuropeCzechModel); |
| 133 |
mProbers[50] = new nsSingleByteCharSetProber(&Ibm852CzechModel); |
| 134 |
|
| 135 |
mProbers[51] = new nsSingleByteCharSetProber(&Windows_1250SlovakModel); |
| 136 |
mProbers[52] = new nsSingleByteCharSetProber(&Iso_8859_2SlovakModel); |
| 137 |
mProbers[53] = new nsSingleByteCharSetProber(&Mac_CentraleuropeSlovakModel); |
| 138 |
mProbers[54] = new nsSingleByteCharSetProber(&Ibm852SlovakModel); |
| 139 |
|
| 140 |
mProbers[55] = new nsSingleByteCharSetProber(&Windows_1250PolishModel); |
| 141 |
mProbers[56] = new nsSingleByteCharSetProber(&Iso_8859_2PolishModel); |
| 142 |
mProbers[57] = new nsSingleByteCharSetProber(&Iso_8859_13PolishModel); |
| 143 |
mProbers[58] = new nsSingleByteCharSetProber(&Iso_8859_16PolishModel); |
| 144 |
mProbers[59] = new nsSingleByteCharSetProber(&Mac_CentraleuropePolishModel); |
| 145 |
mProbers[60] = new nsSingleByteCharSetProber(&Ibm852PolishModel); |
| 146 |
|
| 147 |
mProbers[61] = new nsSingleByteCharSetProber(&Iso_8859_1FinnishModel); |
| 148 |
mProbers[62] = new nsSingleByteCharSetProber(&Iso_8859_4FinnishModel); |
| 149 |
mProbers[63] = new nsSingleByteCharSetProber(&Iso_8859_9FinnishModel); |
| 150 |
mProbers[64] = new nsSingleByteCharSetProber(&Iso_8859_13FinnishModel); |
| 151 |
mProbers[65] = new nsSingleByteCharSetProber(&Iso_8859_15FinnishModel); |
| 152 |
mProbers[66] = new nsSingleByteCharSetProber(&Windows_1252FinnishModel); |
| 153 |
|
| 154 |
mProbers[67] = new nsSingleByteCharSetProber(&Iso_8859_1ItalianModel); |
| 155 |
mProbers[68] = new nsSingleByteCharSetProber(&Iso_8859_3ItalianModel); |
| 156 |
mProbers[69] = new nsSingleByteCharSetProber(&Iso_8859_9ItalianModel); |
| 157 |
mProbers[70] = new nsSingleByteCharSetProber(&Iso_8859_15ItalianModel); |
| 158 |
mProbers[71] = new nsSingleByteCharSetProber(&Windows_1252ItalianModel); |
| 159 |
|
| 160 |
mProbers[72] = new nsSingleByteCharSetProber(&Windows_1250CroatianModel); |
| 161 |
mProbers[73] = new nsSingleByteCharSetProber(&Iso_8859_2CroatianModel); |
| 162 |
mProbers[74] = new nsSingleByteCharSetProber(&Iso_8859_13CroatianModel); |
| 163 |
mProbers[75] = new nsSingleByteCharSetProber(&Iso_8859_16CroatianModel); |
| 164 |
mProbers[76] = new nsSingleByteCharSetProber(&Mac_CentraleuropeCroatianModel); |
| 165 |
mProbers[77] = new nsSingleByteCharSetProber(&Ibm852CroatianModel); |
| 166 |
|
| 167 |
mProbers[78] = new nsSingleByteCharSetProber(&Windows_1252EstonianModel); |
| 168 |
mProbers[79] = new nsSingleByteCharSetProber(&Windows_1257EstonianModel); |
| 169 |
mProbers[80] = new nsSingleByteCharSetProber(&Iso_8859_4EstonianModel); |
| 170 |
mProbers[81] = new nsSingleByteCharSetProber(&Iso_8859_13EstonianModel); |
| 171 |
mProbers[82] = new nsSingleByteCharSetProber(&Iso_8859_15EstonianModel); |
| 172 |
|
| 173 |
mProbers[83] = new nsSingleByteCharSetProber(&Iso_8859_1IrishModel); |
| 174 |
mProbers[84] = new nsSingleByteCharSetProber(&Iso_8859_9IrishModel); |
| 175 |
mProbers[85] = new nsSingleByteCharSetProber(&Iso_8859_15IrishModel); |
| 176 |
mProbers[86] = new nsSingleByteCharSetProber(&Windows_1252IrishModel); |
| 177 |
|
| 178 |
mProbers[87] = new nsSingleByteCharSetProber(&Windows_1250RomanianModel); |
| 179 |
mProbers[88] = new nsSingleByteCharSetProber(&Iso_8859_2RomanianModel); |
| 180 |
mProbers[89] = new nsSingleByteCharSetProber(&Iso_8859_16RomanianModel); |
| 181 |
mProbers[90] = new nsSingleByteCharSetProber(&Ibm852RomanianModel); |
| 182 |
|
| 183 |
mProbers[91] = new nsSingleByteCharSetProber(&Windows_1250SloveneModel); |
| 184 |
mProbers[92] = new nsSingleByteCharSetProber(&Iso_8859_2SloveneModel); |
| 185 |
mProbers[93] = new nsSingleByteCharSetProber(&Iso_8859_16SloveneModel); |
| 186 |
mProbers[94] = new nsSingleByteCharSetProber(&Mac_CentraleuropeSloveneModel); |
| 187 |
mProbers[95] = new nsSingleByteCharSetProber(&Ibm852SloveneModel); |
| 188 |
|
| 189 |
mProbers[96] = new nsSingleByteCharSetProber(&Iso_8859_1SwedishModel); |
| 190 |
mProbers[97] = new nsSingleByteCharSetProber(&Iso_8859_4SwedishModel); |
| 191 |
mProbers[98] = new nsSingleByteCharSetProber(&Iso_8859_9SwedishModel); |
| 192 |
mProbers[99] = new nsSingleByteCharSetProber(&Iso_8859_15SwedishModel); |
| 193 |
mProbers[100] = new nsSingleByteCharSetProber(&Windows_1252SwedishModel); |
| 194 |
|
| 195 |
mProbers[101] = new nsSingleByteCharSetProber(&Iso_8859_15NorwegianModel); |
| 196 |
mProbers[102] = new nsSingleByteCharSetProber(&Iso_8859_1NorwegianModel); |
| 197 |
mProbers[103] = new nsSingleByteCharSetProber(&Windows_1252NorwegianModel); |
| 198 |
mProbers[104] = new nsSingleByteCharSetProber(&Ibm865NorwegianModel); |
| 199 |
|
| 200 |
Reset(); |
| 201 |
} |
| 202 |
|
| 203 |
nsSBCSGroupProber::~nsSBCSGroupProber() |
| 204 |
{ |
| 205 |
for (PRUint32 i = 0; i < NUM_OF_SBCS_PROBERS; i++) |
| 206 |
{ |
| 207 |
delete mProbers[i]; |
| 208 |
} |
| 209 |
} |
| 210 |
|
| 211 |
|
| 212 |
const char* nsSBCSGroupProber::GetCharSetName() |
| 213 |
{ |
| 214 |
//if we have no answer yet |
| 215 |
if (mBestGuess == -1) |
| 216 |
{ |
| 217 |
GetConfidence(); |
| 218 |
//no charset seems positive |
| 219 |
if (mBestGuess == -1) |
| 220 |
//we will use default. |
| 221 |
mBestGuess = 0; |
| 222 |
} |
| 223 |
return mProbers[mBestGuess]->GetCharSetName(); |
| 224 |
} |
| 225 |
|
| 226 |
void nsSBCSGroupProber::Reset(void) |
| 227 |
{ |
| 228 |
mActiveNum = 0; |
| 229 |
for (PRUint32 i = 0; i < NUM_OF_SBCS_PROBERS; i++) |
| 230 |
{ |
| 231 |
if (mProbers[i]) // not null |
| 232 |
{ |
| 233 |
mProbers[i]->Reset(); |
| 234 |
mIsActive[i] = PR_TRUE; |
| 235 |
++mActiveNum; |
| 236 |
} |
| 237 |
else |
| 238 |
mIsActive[i] = PR_FALSE; |
| 239 |
} |
| 240 |
mBestGuess = -1; |
| 241 |
mState = eDetecting; |
| 242 |
} |
| 243 |
|
| 244 |
|
| 245 |
nsProbingState nsSBCSGroupProber::HandleData(const char* aBuf, PRUint32 aLen) |
| 246 |
{ |
| 247 |
nsProbingState st; |
| 248 |
PRUint32 i; |
| 249 |
char *newBuf1 = 0; |
| 250 |
PRUint32 newLen1 = 0; |
| 251 |
|
| 252 |
//apply filter to original buffer, and we got new buffer back |
| 253 |
//depend on what script it is, we will feed them the new buffer |
| 254 |
//we got after applying proper filter |
| 255 |
//this is done without any consideration to KeepEnglishLetters |
| 256 |
//of each prober since as of now, there are no probers here which |
| 257 |
//recognize languages with English characters. |
| 258 |
if (!FilterWithoutEnglishLetters(aBuf, aLen, &newBuf1, newLen1)) |
| 259 |
goto done; |
| 260 |
|
| 261 |
if (newLen1 == 0) |
| 262 |
goto done; // Nothing to see here, move on. |
| 263 |
|
| 264 |
for (i = 0; i < NUM_OF_SBCS_PROBERS; i++) |
| 265 |
{ |
| 266 |
if (!mIsActive[i]) |
| 267 |
continue; |
| 268 |
st = mProbers[i]->HandleData(newBuf1, newLen1); |
| 269 |
if (st == eFoundIt) |
| 270 |
{ |
| 271 |
mBestGuess = i; |
| 272 |
mState = eFoundIt; |
| 273 |
break; |
| 274 |
} |
| 275 |
else if (st == eNotMe) |
| 276 |
{ |
| 277 |
mIsActive[i] = PR_FALSE; |
| 278 |
mActiveNum--; |
| 279 |
if (mActiveNum <= 0) |
| 280 |
{ |
| 281 |
mState = eNotMe; |
| 282 |
break; |
| 283 |
} |
| 284 |
} |
| 285 |
} |
| 286 |
|
| 287 |
done: |
| 288 |
PR_FREEIF(newBuf1); |
| 289 |
|
| 290 |
return mState; |
| 291 |
} |
| 292 |
|
| 293 |
float nsSBCSGroupProber::GetConfidence(void) |
| 294 |
{ |
| 295 |
PRUint32 i; |
| 296 |
float bestConf = 0.0, cf; |
| 297 |
|
| 298 |
switch (mState) |
| 299 |
{ |
| 300 |
case eFoundIt: |
| 301 |
return (float)0.99; //sure yes |
| 302 |
case eNotMe: |
| 303 |
return (float)0.01; //sure no |
| 304 |
default: |
| 305 |
for (i = 0; i < NUM_OF_SBCS_PROBERS; i++) |
| 306 |
{ |
| 307 |
if (!mIsActive[i]) |
| 308 |
continue; |
| 309 |
cf = mProbers[i]->GetConfidence(); |
| 310 |
if (bestConf < cf) |
| 311 |
{ |
| 312 |
bestConf = cf; |
| 313 |
mBestGuess = i; |
| 314 |
} |
| 315 |
} |
| 316 |
} |
| 317 |
return bestConf; |
| 318 |
} |
| 319 |
|
| 320 |
#ifdef DEBUG_chardet |
| 321 |
void nsSBCSGroupProber::DumpStatus() |
| 322 |
{ |
| 323 |
PRUint32 i; |
| 324 |
float cf; |
| 325 |
|
| 326 |
cf = GetConfidence(); |
| 327 |
printf(" SBCS Group Prober --------begin status \r\n"); |
| 328 |
for (i = 0; i < NUM_OF_SBCS_PROBERS; i++) |
| 329 |
{ |
| 330 |
if (!mIsActive[i]) |
| 331 |
printf(" inactive: [%s] (i.e. confidence is too low).\r\n", mProbers[i]->GetCharSetName()); |
| 332 |
else |
| 333 |
mProbers[i]->DumpStatus(); |
| 334 |
} |
| 335 |
printf(" SBCS Group found best match [%s] confidence %f.\r\n", |
| 336 |
mProbers[mBestGuess]->GetCharSetName(), cf); |
| 337 |
} |
| 338 |
#endif |