Debian Codegraph @

src/LangModels/LangDanishModel.cpp

raw
1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* ***** BEGIN LICENSE BLOCK *****
3 * Version: MPL 1.1/GPL 2.0/LGPL 2.1
4 *
5 * The contents of this file are subject to the Mozilla Public License Version
6 * 1.1 (the "License"); you may not use this file except in compliance with
7 * the License. You may obtain a copy of the License at
8 * http://www.mozilla.org/MPL/
9 *
10 * Software distributed under the License is distributed on an "AS IS" basis,
11 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
12 * for the specific language governing rights and limitations under the
13 * License.
14 *
15 * The Original Code is Mozilla Communicator client code.
16 *
17 * The Initial Developer of the Original Code is
18 * Netscape Communications Corporation.
19 * Portions created by the Initial Developer are Copyright (C) 1998
20 * the Initial Developer. All Rights Reserved.
21 *
22 * Contributor(s):
23 *
24 * Alternatively, the contents of this file may be used under the terms of
25 * either the GNU General Public License Version 2 or later (the "GPL"), or
26 * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
27 * in which case the provisions of the GPL or the LGPL are applicable instead
28 * of those above. If you wish to allow use of your version of this file only
29 * under the terms of either the GPL or the LGPL, and not to allow others to
30 * use your version of this file under the terms of the MPL, indicate your
31 * decision by deleting the provisions above and replace them with the notice
32 * and other provisions required by the GPL or the LGPL. If you do not delete
33 * the provisions above, a recipient may use your version of this file under
34 * the terms of any one of the MPL, the GPL or the LGPL.
35 *
36 * ***** END LICENSE BLOCK ***** */
37
38 #include "../nsSBCharSetProber.h"
39
40 /********* Language model for: Danish *********/
41
42 /**
43 * Generated by BuildLangModel.py
44 * On: 2022-11-30 19:41:17.519380
45 **/
46
47 /* Character Mapping Table:
48 * ILL: illegal character.
49 * CTR: control character specific to the charset.
50 * RET: carriage/return.
51 * SYM: symbol (punctuation) that does not belong to word.
52 * NUM: 0 - 9.
53 *
54 * Other characters are ordered by probabilities
55 * (0 is the most common character in the language).
56 *
57 * Orders are generic to a language. So the codepoint with order X in
58 * CHARSET1 maps to the same character as the codepoint with the same
59 * order X in CHARSET2 for the same language.
60 * As such, it is possible to get missing order. For instance the
61 * ligature of 'o' and 'e' exists in ISO-8859-15 but not in ISO-8859-1
62 * even though they are both used for French. Same for the euro sign.
63 */
64 static const unsigned char Iso_8859_15_CharToOrderMap[] =
65 {
66 CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,RET,CTR,CTR,RET,CTR,CTR, /* 0X */
67 CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR, /* 1X */
68 SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM, /* 2X */
69 NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,SYM,SYM,SYM,SYM,SYM,SYM, /* 3X */
70 SYM, 4, 16, 24, 7, 0, 13, 10, 18, 5, 23, 11, 8, 12, 2, 9, /* 4X */
71 17, 30, 1, 6, 3, 15, 14, 25, 27, 22, 26,SYM,SYM,SYM,SYM,SYM, /* 5X */
72 SYM, 4, 16, 24, 7, 0, 13, 10, 18, 5, 23, 11, 8, 12, 2, 9, /* 6X */
73 17, 30, 1, 6, 3, 15, 14, 25, 27, 22, 26,SYM,SYM,SYM,SYM,CTR, /* 7X */
74 CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR, /* 8X */
75 CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR, /* 9X */
76 SYM,SYM,SYM,SYM,SYM,SYM, 50,SYM, 50,SYM,SYM,SYM,SYM,SYM,SYM,SYM, /* AX */
77 SYM,SYM,SYM,SYM, 60, 57,SYM,SYM, 61,SYM,SYM,SYM, 43, 43, 62,SYM, /* BX */
78 39, 32, 44, 53, 36, 21, 20, 42, 38, 28, 63, 46, 64, 35, 47, 52, /* CX */
79 31, 48, 58, 29, 49, 59, 34,SYM, 19, 65, 37, 66, 33, 40, 55, 41, /* DX */
80 39, 32, 44, 53, 36, 21, 20, 42, 38, 28, 67, 46, 68, 35, 47, 52, /* EX */
81 31, 48, 58, 29, 49, 59, 34,SYM, 19, 69, 37, 70, 33, 40, 55, 71, /* FX */
82 };
83 /*X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 XA XB XC XD XE XF */
84
85 static const unsigned char Iso_8859_1_CharToOrderMap[] =
86 {
87 CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,RET,CTR,CTR,RET,CTR,CTR, /* 0X */
88 CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR, /* 1X */
89 SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM, /* 2X */
90 NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,SYM,SYM,SYM,SYM,SYM,SYM, /* 3X */
91 SYM, 4, 16, 24, 7, 0, 13, 10, 18, 5, 23, 11, 8, 12, 2, 9, /* 4X */
92 17, 30, 1, 6, 3, 15, 14, 25, 27, 22, 26,SYM,SYM,SYM,SYM,SYM, /* 5X */
93 SYM, 4, 16, 24, 7, 0, 13, 10, 18, 5, 23, 11, 8, 12, 2, 9, /* 6X */
94 17, 30, 1, 6, 3, 15, 14, 25, 27, 22, 26,SYM,SYM,SYM,SYM,CTR, /* 7X */
95 CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR, /* 8X */
96 CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR, /* 9X */
97 SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM, /* AX */
98 SYM,SYM,SYM,SYM,SYM, 57,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM, /* BX */
99 39, 32, 44, 53, 36, 21, 20, 42, 38, 28, 72, 46, 73, 35, 47, 52, /* CX */
100 31, 48, 58, 29, 49, 59, 34,SYM, 19, 74, 37, 75, 33, 40, 55, 41, /* DX */
101 39, 32, 44, 53, 36, 21, 20, 42, 38, 28, 76, 46, 77, 35, 47, 52, /* EX */
102 31, 48, 58, 29, 49, 59, 34,SYM, 19, 78, 37, 79, 33, 40, 55, 80, /* FX */
103 };
104 /*X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 XA XB XC XD XE XF */
105
106 static const unsigned char Windows_1252_CharToOrderMap[] =
107 {
108 CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,RET,CTR,CTR,RET,CTR,CTR, /* 0X */
109 CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR, /* 1X */
110 SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM, /* 2X */
111 NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,SYM,SYM,SYM,SYM,SYM,SYM, /* 3X */
112 SYM, 4, 16, 24, 7, 0, 13, 10, 18, 5, 23, 11, 8, 12, 2, 9, /* 4X */
113 17, 30, 1, 6, 3, 15, 14, 25, 27, 22, 26,SYM,SYM,SYM,SYM,SYM, /* 5X */
114 SYM, 4, 16, 24, 7, 0, 13, 10, 18, 5, 23, 11, 8, 12, 2, 9, /* 6X */
115 17, 30, 1, 6, 3, 15, 14, 25, 27, 22, 26,SYM,SYM,SYM,SYM,CTR, /* 7X */
116 SYM,ILL,SYM, 81,SYM,SYM,SYM,SYM,SYM,SYM, 50,SYM, 43,ILL, 82,ILL, /* 8X */
117 ILL,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM, 50,SYM, 43,ILL, 83, 84, /* 9X */
118 SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM, /* AX */
119 SYM,SYM,SYM,SYM,SYM, 57,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM, /* BX */
120 39, 32, 44, 53, 36, 21, 20, 42, 38, 28, 85, 46, 86, 35, 47, 52, /* CX */
121 31, 48, 58, 29, 49, 59, 34,SYM, 19, 87, 37, 88, 33, 40, 55, 41, /* DX */
122 39, 32, 44, 53, 36, 21, 20, 42, 38, 28, 89, 46, 90, 35, 47, 52, /* EX */
123 31, 48, 58, 29, 49, 59, 34,SYM, 19, 91, 37, 92, 33, 40, 55, 93, /* FX */
124 };
125 /*X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 XA XB XC XD XE XF */
126
127 static const unsigned char Ibm865_CharToOrderMap[] =
128 {
129 CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,RET,CTR,CTR,RET,CTR,CTR, /* 0X */
130 CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR,CTR, /* 1X */
131 SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM, /* 2X */
132 NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,NUM,SYM,SYM,SYM,SYM,SYM,SYM, /* 3X */
133 SYM, 4, 16, 24, 7, 0, 13, 10, 18, 5, 23, 11, 8, 12, 2, 9, /* 4X */
134 17, 30, 1, 6, 3, 15, 14, 25, 27, 22, 26,SYM,SYM,SYM,SYM,SYM, /* 5X */
135 SYM, 4, 16, 24, 7, 0, 13, 10, 18, 5, 23, 11, 8, 12, 2, 9, /* 6X */
136 17, 30, 1, 6, 3, 15, 14, 25, 27, 22, 26,SYM,SYM,SYM,SYM,CTR, /* 7X */
137 42, 33, 28, 44, 36, 39, 21, 42, 94, 46, 38, 52, 47, 95, 36, 21, /* 8X */
138 28, 20, 20, 49, 34, 58, 96, 97, 98, 34, 33, 19,SYM, 19,SYM,SYM, /* 9X */
139 32, 35, 29, 37, 48, 48,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM, /* AX */
140 SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM, /* BX */
141 SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM, /* CX */
142 SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM, /* DX */
143 45, 41, 99, 56,100,101, 57, 54,102,103,104,105,106,107, 51,SYM, /* EX */
144 SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM,SYM, /* FX */
145 };
146 /*X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 XA XB XC XD XE XF */
147
148
149 /* Model Table:
150 * Total sequences: 1065
151 * First 512 sequences: 0.9958348814328518
152 * Next 512 sequences (512-1024): 0.0041324290837536455
153 * Rest: 3.268948339453948e-05
154 * Negative sequences: TODO
155 */
156 static const PRUint8 DanishLangModel[] =
157 {
158 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,3,3,3,3,2,3,0,0,2,
159 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,2,2,2,2,2,2,
160 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,2,2,2,2,
161 3,3,3,3,3,3,3,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,1,2,2,1,
162 3,3,3,3,3,3,3,3,3,2,3,3,3,3,3,3,3,3,3,0,2,2,3,3,3,3,3,3,2,0,2,
163 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,3,3,2,3,3,2,0,2,
164 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,0,2,2,2,
165 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,0,2,2,2,
166 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,0,2,2,2,
167 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,1,2,3,3,3,3,2,2,2,0,2,
168 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,3,3,3,3,2,2,0,0,2,0,0,
169 3,3,3,3,3,3,3,2,3,3,2,3,3,3,3,3,2,2,3,3,3,3,3,3,1,2,2,1,2,0,0,
170 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,2,2,2,2,2,2,
171 3,3,3,3,3,3,3,3,3,3,3,2,2,3,3,3,3,2,3,3,3,3,3,3,2,0,2,2,2,2,2,
172 3,3,3,3,3,3,3,3,3,3,2,2,3,2,2,3,2,2,2,3,3,3,2,3,2,0,0,0,1,1,0,
173 3,3,3,3,3,3,3,3,3,2,3,3,3,3,3,2,3,3,3,2,2,0,2,2,3,2,2,3,0,0,2,
174 3,3,3,3,3,3,3,2,3,3,2,2,2,2,2,3,3,2,2,3,3,3,3,3,2,0,2,0,2,2,0,
175 3,3,3,3,3,3,3,3,3,3,3,2,2,3,2,3,3,3,3,3,3,3,3,2,2,0,1,0,2,2,0,
176 3,3,3,3,3,3,3,2,3,3,1,2,3,2,3,3,2,2,2,3,3,3,3,3,2,3,0,0,2,2,1,
177 3,3,3,3,0,2,3,3,3,2,3,3,3,2,3,2,3,2,2,0,0,0,2,3,0,2,1,0,0,0,0,
178 2,3,3,3,2,3,3,3,3,3,3,3,3,3,3,2,3,2,2,0,0,0,0,0,2,0,0,0,0,0,0,
179 3,3,3,3,0,0,3,3,3,2,2,3,2,2,3,0,3,2,2,0,0,0,0,0,0,0,0,0,0,0,0,
180 3,3,3,3,3,2,3,3,3,3,3,3,3,2,3,2,3,3,3,2,0,0,2,2,2,2,2,0,0,0,0,
181 3,3,3,2,3,3,3,3,3,3,2,2,2,2,2,3,2,2,2,3,3,2,3,0,2,0,0,0,0,2,0,
182 3,3,2,3,3,3,2,2,3,3,2,3,2,2,2,3,2,2,3,0,2,0,3,2,3,0,2,2,2,2,2,
183 3,2,2,2,3,3,2,2,2,3,0,2,2,2,0,2,2,0,2,0,2,0,2,2,2,2,2,0,0,0,2,
184 3,2,2,2,3,3,2,2,2,3,2,2,2,2,0,2,2,2,2,0,0,0,2,2,0,2,3,0,0,0,0,
185 3,2,1,2,2,2,2,2,2,2,0,2,1,2,2,0,0,2,0,0,0,0,2,0,2,2,0,2,0,0,0,
186 2,2,3,2,2,0,2,2,2,2,2,0,2,2,2,2,2,1,2,0,0,0,0,0,1,0,2,0,0,0,0,
187 0,3,2,2,2,0,2,0,2,0,2,2,0,2,2,0,0,2,2,0,0,0,0,0,0,2,0,0,0,0,0,
188 0,0,0,0,2,1,0,0,0,0,0,0,0,0,2,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
189 };
190
191
192 const SequenceModel Iso_8859_15DanishModel =
193 {
194 Iso_8859_15_CharToOrderMap,
195 DanishLangModel,
196 31,
197 (float)0.9958348814328518,
198 PR_TRUE,
199 "ISO-8859-15"
200 };
201
202 const SequenceModel Iso_8859_1DanishModel =
203 {
204 Iso_8859_1_CharToOrderMap,
205 DanishLangModel,
206 31,
207 (float)0.9958348814328518,
208 PR_TRUE,
209 "ISO-8859-1"
210 };
211
212 const SequenceModel Windows_1252DanishModel =
213 {
214 Windows_1252_CharToOrderMap,
215 DanishLangModel,
216 31,
217 (float)0.9958348814328518,
218 PR_TRUE,
219 "WINDOWS-1252"
220 };
221
222 const SequenceModel Ibm865DanishModel =
223 {
224 Ibm865_CharToOrderMap,
225 DanishLangModel,
226 31,
227 (float)0.9958348814328518,
228 PR_TRUE,
229 "IBM865"
230 };