Debian Codegraph @

src/CharDistribution.h

raw
1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* ***** BEGIN LICENSE BLOCK *****
3 * Version: MPL 1.1/GPL 2.0/LGPL 2.1
4 *
5 * The contents of this file are subject to the Mozilla Public License Version
6 * 1.1 (the "License"); you may not use this file except in compliance with
7 * the License. You may obtain a copy of the License at
8 * http://www.mozilla.org/MPL/
9 *
10 * Software distributed under the License is distributed on an "AS IS" basis,
11 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
12 * for the specific language governing rights and limitations under the
13 * License.
14 *
15 * The Original Code is Mozilla Communicator client code.
16 *
17 * The Initial Developer of the Original Code is
18 * Netscape Communications Corporation.
19 * Portions created by the Initial Developer are Copyright (C) 1998
20 * the Initial Developer. All Rights Reserved.
21 *
22 * Contributor(s):
23 *
24 * Alternatively, the contents of this file may be used under the terms of
25 * either the GNU General Public License Version 2 or later (the "GPL"), or
26 * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
27 * in which case the provisions of the GPL or the LGPL are applicable instead
28 * of those above. If you wish to allow use of your version of this file only
29 * under the terms of either the GPL or the LGPL, and not to allow others to
30 * use your version of this file under the terms of the MPL, indicate your
31 * decision by deleting the provisions above and replace them with the notice
32 * and other provisions required by the GPL or the LGPL. If you do not delete
33 * the provisions above, a recipient may use your version of this file under
34 * the terms of any one of the MPL, the GPL or the LGPL.
35 *
36 * ***** END LICENSE BLOCK ***** */
37
38 #ifndef CharDistribution_h__
39 #define CharDistribution_h__
40
41 #include "nscore.h"
42
43 #define ENOUGH_DATA_THRESHOLD 1024
44
45 #define MINIMUM_DATA_THRESHOLD 4
46
47 class CharDistributionAnalysis
48 {
49 public:
50 CharDistributionAnalysis() {Reset(PR_FALSE);}
51
52 //feed a block of data and do distribution analysis
53 void HandleData(const char* aBuf, PRUint32 aLen) {}
54
55 //Feed a character with known length
56 void HandleOneChar(const char* aStr, PRUint32 aCharLen)
57 {
58 PRInt32 order;
59
60 //we only care about 2-bytes character in our distribution analysis
61 order = (aCharLen == 2) ? GetOrder(aStr) : -1;
62
63 if (order >= 0)
64 {
65 mTotalChars++;
66 //order is valid
67 if ((PRUint32)order < mTableSize)
68 {
69 if (512 > mCharToFreqOrder[order])
70 mFreqChars++;
71 }
72 }
73 }
74
75 //return confidence base on existing data
76 float GetConfidence(void);
77
78 //Reset analyser, clear any state
79 void Reset(PRBool aIsPreferredLanguage)
80 {
81 mDone = PR_FALSE;
82 mTotalChars = 0;
83 mFreqChars = 0;
84 mDataThreshold = aIsPreferredLanguage ? 0 : MINIMUM_DATA_THRESHOLD;
85 }
86
87 //This function is for future extension. Caller can use this function to control
88 //analyser's behavior
89 void SetOpion(){}
90
91 //It is not necessary to receive all data to draw conclusion. For charset detection,
92 // certain amount of data is enough
93 PRBool GotEnoughData() {return mTotalChars > ENOUGH_DATA_THRESHOLD;}
94
95 protected:
96 //we do not handle character base on its original encoding string, but
97 //convert this encoding string to a number, here called order.
98 //This allow multiple encoding of a language to share one frequency table
99 virtual PRInt32 GetOrder(const char* str) {return -1;}
100
101 //If this flag is set to PR_TRUE, detection is done and conclusion has been made
102 PRBool mDone;
103
104 //The number of characters whose frequency order is less than 512
105 PRUint32 mFreqChars;
106
107 //Total character encounted.
108 PRUint32 mTotalChars;
109
110 //Number of hi-byte characters needed to trigger detection
111 PRUint32 mDataThreshold;
112
113 //Mapping table to get frequency order from char order (get from GetOrder())
114 const PRInt16 *mCharToFreqOrder;
115
116 //Size of above table
117 PRUint32 mTableSize;
118
119 //This is a constant value varies from language to language, it is used in
120 //calculating confidence. See my paper for further detail.
121 float mTypicalDistributionRatio;
122 };
123
124
125 class EUCTWDistributionAnalysis: public CharDistributionAnalysis
126 {
127 public:
128 EUCTWDistributionAnalysis();
129 protected:
130
131 //for EUC-TW encoding, we are interested
132 // first byte range: 0xc4 -- 0xfe
133 // second byte range: 0xa1 -- 0xfe
134 //no validation needed here. State machine has done that
135 PRInt32 GetOrder(const char* str)
136 { if ((unsigned char)*str >= (unsigned char)0xc4)
137 return 94*((unsigned char)str[0]-(unsigned char)0xc4) + (unsigned char)str[1] - (unsigned char)0xa1;
138 else
139 return -1;
140 }
141 };
142
143
144 class EUCKRDistributionAnalysis : public CharDistributionAnalysis
145 {
146 public:
147 EUCKRDistributionAnalysis();
148 protected:
149 //for euc-KR encoding, we are interested
150 // first byte range: 0xb0 -- 0xfe
151 // second byte range: 0xa1 -- 0xfe
152 //no validation needed here. State machine has done that
153 PRInt32 GetOrder(const char* str)
154 { if ((unsigned char)*str >= (unsigned char)0xb0)
155 return 94*((unsigned char)str[0]-(unsigned char)0xb0) + (unsigned char)str[1] - (unsigned char)0xa1;
156 else
157 return -1;
158 }
159 };
160
161 class GB2312DistributionAnalysis : public CharDistributionAnalysis
162 {
163 public:
164 GB2312DistributionAnalysis();
165 protected:
166 //for GB2312 encoding, we are interested
167 // first byte range: 0xb0 -- 0xfe
168 // second byte range: 0xa1 -- 0xfe
169 //no validation needed here. State machine has done that
170 PRInt32 GetOrder(const char* str)
171 { if ((unsigned char)*str >= (unsigned char)0xb0 && (unsigned char)str[1] >= (unsigned char)0xa1)
172 return 94*((unsigned char)str[0]-(unsigned char)0xb0) + (unsigned char)str[1] - (unsigned char)0xa1;
173 else
174 return -1;
175 }
176 };
177
178
179 class Big5DistributionAnalysis : public CharDistributionAnalysis
180 {
181 public:
182 Big5DistributionAnalysis();
183 protected:
184 //for big5 encoding, we are interested
185 // first byte range: 0xa4 -- 0xfe
186 // second byte range: 0x40 -- 0x7e , 0xa1 -- 0xfe
187 //no validation needed here. State machine has done that
188 PRInt32 GetOrder(const char* str)
189 { if ((unsigned char)*str >= (unsigned char)0xa4)
190 if ((unsigned char)str[1] >= (unsigned char)0xa1)
191 return 157*((unsigned char)str[0]-(unsigned char)0xa4) + (unsigned char)str[1] - (unsigned char)0xa1 +63;
192 else
193 return 157*((unsigned char)str[0]-(unsigned char)0xa4) + (unsigned char)str[1] - (unsigned char)0x40;
194 else
195 return -1;
196 }
197 };
198
199 class SJISDistributionAnalysis : public CharDistributionAnalysis
200 {
201 public:
202 SJISDistributionAnalysis();
203 protected:
204 //for sjis encoding, we are interested
205 // first byte range: 0x81 -- 0x9f , 0xe0 -- 0xfe
206 // second byte range: 0x40 -- 0x7e, 0x81 -- oxfe
207 //no validation needed here. State machine has done that
208 PRInt32 GetOrder(const char* str)
209 {
210 PRInt32 order;
211 if ((unsigned char)*str >= (unsigned char)0x81 && (unsigned char)*str <= (unsigned char)0x9f)
212 order = 188 * ((unsigned char)str[0]-(unsigned char)0x81);
213 else if ((unsigned char)*str >= (unsigned char)0xe0 && (unsigned char)*str <= (unsigned char)0xef)
214 order = 188 * ((unsigned char)str[0]-(unsigned char)0xe0 + 31);
215 else
216 return -1;
217 order += (unsigned char)*(str+1) - 0x40;
218 if ((unsigned char)str[1] > (unsigned char)0x7f)
219 order--;
220 return order;
221 }
222 };
223
224 class EUCJPDistributionAnalysis : public CharDistributionAnalysis
225 {
226 public:
227 EUCJPDistributionAnalysis();
228 protected:
229 //for euc-JP encoding, we are interested
230 // first byte range: 0xa0 -- 0xfe
231 // second byte range: 0xa1 -- 0xfe
232 //no validation needed here. State machine has done that
233 PRInt32 GetOrder(const char* str)
234 { if ((unsigned char)*str >= (unsigned char)0xa0)
235 return 94*((unsigned char)str[0]-(unsigned char)0xa1) + (unsigned char)str[1] - (unsigned char)0xa1;
236 else
237 return -1;
238 }
239 };
240
241 #endif //CharDistribution_h__
242