Debian Codegraph @

src/nsSBCharSetProber.cpp

raw
1 /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* ***** BEGIN LICENSE BLOCK *****
3 * Version: MPL 1.1/GPL 2.0/LGPL 2.1
4 *
5 * The contents of this file are subject to the Mozilla Public License Version
6 * 1.1 (the "License"); you may not use this file except in compliance with
7 * the License. You may obtain a copy of the License at
8 * http://www.mozilla.org/MPL/
9 *
10 * Software distributed under the License is distributed on an "AS IS" basis,
11 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
12 * for the specific language governing rights and limitations under the
13 * License.
14 *
15 * The Original Code is Mozilla Universal charset detector code.
16 *
17 * The Initial Developer of the Original Code is
18 * Netscape Communications Corporation.
19 * Portions created by the Initial Developer are Copyright (C) 2001
20 * the Initial Developer. All Rights Reserved.
21 *
22 * Contributor(s):
23 * Shy Shalom <[email protected]>
24 *
25 * Alternatively, the contents of this file may be used under the terms of
26 * either the GNU General Public License Version 2 or later (the "GPL"), or
27 * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
28 * in which case the provisions of the GPL or the LGPL are applicable instead
29 * of those above. If you wish to allow use of your version of this file only
30 * under the terms of either the GPL or the LGPL, and not to allow others to
31 * use your version of this file under the terms of the MPL, indicate your
32 * decision by deleting the provisions above and replace them with the notice
33 * and other provisions required by the GPL or the LGPL. If you do not delete
34 * the provisions above, a recipient may use your version of this file under
35 * the terms of any one of the MPL, the GPL or the LGPL.
36 *
37 * ***** END LICENSE BLOCK ***** */
38 #include <stdio.h>
39 #include "nsSBCharSetProber.h"
40
41 nsProbingState nsSingleByteCharSetProber::HandleData(const char* aBuf, PRUint32 aLen)
42 {
43 unsigned char order;
44
45 for (PRUint32 i = 0; i < aLen; i++)
46 {
47 order = mModel->charToOrderMap[(unsigned char)aBuf[i]];
48
49 if (order < SYMBOL_CAT_ORDER)
50 {
51 mTotalChar++;
52 }
53 else if (order == ILL)
54 {
55 /* When encountering an illegal codepoint, no need
56 * to continue analyzing data. */
57 mState = eNotMe;
58 break;
59 }
60 else if (order == CTR)
61 {
62 mCtrlChar++;
63 }
64 if (order < mModel->freqCharCount)
65 {
66 mFreqChar++;
67
68 if (mLastOrder < mModel->freqCharCount)
69 {
70 mTotalSeqs++;
71 if (!mReversed)
72 ++(mSeqCounters[mModel->precedenceMatrix[mLastOrder*mModel->freqCharCount+order]]);
73 else // reverse the order of the letters in the lookup
74 ++(mSeqCounters[mModel->precedenceMatrix[order*mModel->freqCharCount+mLastOrder]]);
75 }
76 }
77 mLastOrder = order;
78 }
79
80 if (mState == eDetecting)
81 if (mTotalSeqs > SB_ENOUGH_REL_THRESHOLD)
82 {
83 float cf = GetConfidence();
84 if (cf > POSITIVE_SHORTCUT_THRESHOLD)
85 mState = eFoundIt;
86 else if (cf < NEGATIVE_SHORTCUT_THRESHOLD)
87 mState = eNotMe;
88 }
89
90 return mState;
91 }
92
93 void nsSingleByteCharSetProber::Reset(void)
94 {
95 mState = eDetecting;
96 mLastOrder = 255;
97 for (PRUint32 i = 0; i < NUMBER_OF_SEQ_CAT; i++)
98 mSeqCounters[i] = 0;
99 mTotalSeqs = 0;
100 mTotalChar = 0;
101 mCtrlChar = 0;
102 mFreqChar = 0;
103 }
104
105 //#define NEGATIVE_APPROACH 1
106
107 float nsSingleByteCharSetProber::GetConfidence(void)
108 {
109 #ifdef NEGATIVE_APPROACH
110 if (mTotalSeqs > 0)
111 if (mTotalSeqs > mSeqCounters[NEGATIVE_CAT]*10 )
112 return ((float)(mTotalSeqs - mSeqCounters[NEGATIVE_CAT]*10))/mTotalSeqs * mFreqChar / mTotalChar;
113 return (float)0.01;
114 #else //POSITIVE_APPROACH
115 float r;
116
117 if (mTotalSeqs > 0) {
118 r = ((float)1.0) * mSeqCounters[POSITIVE_CAT] / mTotalSeqs / mModel->mTypicalPositiveRatio;
119 /* Multiply by a ratio of positive sequences per characters.
120 * This would help in particular to distinguish close winners.
121 * Indeed if you add a letter, you'd expect the positive sequence count
122 * to increase as well. If it doesn't, it may mean that this new codepoint
123 * may not have been a letter, but instead a symbol (or some other
124 * character). This could make the difference between very closely related
125 * charsets used for the same language.
126 */
127 r = r * (mSeqCounters[POSITIVE_CAT] + (float) mSeqCounters[PROBABLE_CAT] / 4) / mTotalChar;
128 /* The more control characters (proportionnaly to the size of the text), the
129 * less confident we become in the current charset.
130 */
131 r = r * (mTotalChar - mCtrlChar) / mTotalChar;
132 r = r*mFreqChar/mTotalChar;
133 if (r >= (float)1.00)
134 r = (float)0.99;
135 return r;
136 }
137 return (float)0.01;
138 #endif
139 }
140
141 const char* nsSingleByteCharSetProber::GetCharSetName()
142 {
143 if (!mNameProber)
144 return mModel->charsetName;
145 return mNameProber->GetCharSetName();
146 }
147
148 #ifdef DEBUG_chardet
149 void nsSingleByteCharSetProber::DumpStatus()
150 {
151 printf(" SBCS: %1.3f [%s]\r\n", GetConfidence(), GetCharSetName());
152 }
153 #endif