Debian Codegraph @

script/BuildLangModel.py

raw
1 #!/bin/python3
2 # -*- coding: utf-8 -*-
3
4 # ##### BEGIN LICENSE BLOCK #####
5 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
6 #
7 # The contents of this file are subject to the Mozilla Public License Version
8 # 1.1 (the "License"); you may not use this file except in compliance with
9 # the License. You may obtain a copy of the License at
10 # http://www.mozilla.org/MPL/
11 #
12 # Software distributed under the License is distributed on an "AS IS" basis,
13 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
14 # for the specific language governing rights and limitations under the
15 # License.
16 #
17 # The Original Code is Mozilla Universal charset detector code.
18 #
19 # The Initial Developer of the Original Code is
20 # Netscape Communications Corporation.
21 # Portions created by the Initial Developer are Copyright (C) 2001
22 # the Initial Developer. All Rights Reserved.
23 #
24 # Contributor(s):
25 # Jehan <[email protected]>
26 #
27 # Alternatively, the contents of this file may be used under the terms of
28 # either the GNU General Public License Version 2 or later (the "GPL"), or
29 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
30 # in which case the provisions of the GPL or the LGPL are applicable instead
31 # of those above. If you wish to allow use of your version of this file only
32 # under the terms of either the GPL or the LGPL, and not to allow others to
33 # use your version of this file under the terms of the MPL, indicate your
34 # decision by deleting the provisions above and replace them with the notice
35 # and other provisions required by the GPL or the LGPL. If you do not delete
36 # the provisions above, a recipient may use your version of this file under
37 # the terms of any one of the MPL, the GPL or the LGPL.
38 #
39 # ##### END LICENSE BLOCK #####
40
41 # Third party modules.
42 import unicodedata
43 import subprocess
44 import wikipedia
45 import importlib
46 import optparse
47 import datetime
48 import operator
49 import requests
50 import sys
51 import re
52 import os
53 import random
54
55 # Custom modules.
56 import charsets.db
57 from charsets.codepoints import *
58
59 # Command line processing.
60 usage = 'Usage: {} <LANG-CODE>\n' \
61 '\nEx: `{} fr`'.format(__file__, __file__)
62
63 description = "Internal tool for uchardet to generate language data."
64 cmdline = optparse.OptionParser(usage, description = description)
65 cmdline.add_option('--max-page',
66 help = 'Maximum number of Wikipedia pages to parse (useful for debugging).',
67 action = 'store', type = 'int', dest = 'max_page', default = None)
68 cmdline.add_option('--max-depth',
69 help = 'Maximum depth when following links from start page (default: 2).',
70 action = 'store', type = 'int',
71 dest = 'max_depth', default = 2)
72 (options, langs) = cmdline.parse_args()
73 if len(langs) < 1:
74 print("Please select at least one language code.\n")
75 exit(1)
76 if len(langs) > 1:
77 print("This script is meant to generate data for one language at a time.\n")
78 exit(1)
79 lang = langs[0]
80
81 # Load the language data.
82 sys_path_backup = sys.path
83 current_dir = os.path.dirname(os.path.realpath(__file__))
84 sys.path = [current_dir + '/langs']
85
86 try:
87 lang = importlib.import_module(lang.lower())
88 except ImportError:
89 print('Unknown language code "{}": '
90 'file "langs/{}.py" does not exist.'.format(lang, lang.lower()))
91 exit(1)
92 sys.path = sys_path_backup
93
94 charsets = charsets.db.load(lang.charsets)
95
96 if not hasattr(lang, 'start_pages') or lang.start_pages is None or \
97 lang.start_pages == []:
98 # Let's start with the main page, assuming it should have links
99 # to relevant pages. In locale wikipedia, this page is usually redirected
100 # to a relevant page.
101 print("Warning: no `start_pages` set for '{}'. Using ['Main_Page'].\n"
102 " If you don't get good data, it is advised to set a "
103 "start_pages` variable yourself.".format(lang.code))
104 lang.start_pages = ['Main_Page']
105 if not hasattr(lang, 'wikipedia_code') or lang.wikipedia_code is None:
106 lang.wikipedia_code = lang.code
107 if not hasattr(lang, 'clean_wikipedia_content') or lang.clean_wikipedia_content is None:
108 lang.clean_wikipedia_content = None
109 if hasattr(lang, 'case_mapping'):
110 lang.case_mapping = bool(lang.case_mapping)
111 else:
112 lang.case_mapping = False
113 if not hasattr(lang, 'custom_case_mapping'):
114 lang.custom_case_mapping = None
115 if not hasattr(lang, 'alphabet') or lang.alphabet is None:
116 lang.alphabet = None
117
118 def local_lowercase(text, lang):
119 lowercased = ''
120 for l in text:
121 if lang.custom_case_mapping is not None and \
122 l in lang.custom_case_mapping:
123 lowercased += lang.custom_case_mapping[l]
124 elif l.isupper() and \
125 lang.case_mapping and \
126 len(unicodedata.normalize('NFC', l.lower())) == 1:
127 lowercased += l.lower()
128 else:
129 lowercased += l
130 return lowercased
131
132 if lang.alphabet is not None:
133 # Allowing to provide an alphabet in string format rather than list.
134 lang.alphabet = list(lang.alphabet)
135 if lang.use_ascii:
136 lang.alphabet += [chr(l) for l in range(65, 91)] + [chr(l) for l in range(97, 123)]
137 if lang.case_mapping or lang.custom_case_mapping is not None:
138 lang.alphabet = [local_lowercase(l, lang) for l in lang.alphabet]
139 #alphabet = []
140 #for l in lang.alphabet:
141 #if l.isupper() and \
142 #lang.custom_case_mapping is not None and \
143 #l in lang.custom_case_mapping:
144 #alphabet.append(lang.custom_case_mapping[l])
145 #elif l.isupper() and \
146 #lang.case_mapping and \
147 #len(unicodedata.normalize('NFC', l.lower())) == 1:
148 #alphabet.append(l.lower())
149 #else:
150 #alphabet.append(l)
151 lang.alphabet = list(set(lang.alphabet))
152
153 # Starting processing.
154 wikipedia.set_lang(lang.wikipedia_code)
155
156 visited_pages = []
157
158 # The full list of letter characters.
159 # The key is the unicode codepoint,
160 # and the value is the occurrence count.
161 characters = {}
162 # Sequence of letters.
163 # The key is the couple (char1, char2) in unicode codepoint,
164 # the value is the occurrence count.
165 sequences = {}
166 prev_char = None
167
168 def process_text(content, lang):
169 global charsets
170 global characters
171 global sequences
172 global prev_char
173
174 if lang.clean_wikipedia_content is not None:
175 content = lang.clean_wikipedia_content(content)
176 # Clean out the Wikipedia syntax for titles.
177 content = re.sub(r'(=+) *([^=]+) *\1',
178 r'\2', content)
179 # Clean multiple spaces. Newlines and such are normalized to spaces,
180 # since they have basically a similar role in the purpose of uchardet.
181 content = re.sub(r'\s+', ' ', content)
182
183 if lang.case_mapping or lang.custom_case_mapping is not None:
184 content = local_lowercase(content, lang)
185
186 # In python 3, strings are UTF-8.
187 # Looping through them return expected characters.
188 for char in content:
189 is_letter = False
190 if ord(char) in characters:
191 characters[ord(char)] += 1
192 is_letter = True
193 else:
194 # We save the character if it is at least in one of the
195 # language encodings and its not a special character.
196 for charset in charsets:
197 # Does the character exist in the charset?
198 try:
199 codepoint = char.encode(charset, 'ignore')
200 except LookupError:
201 # unknown encoding. Use iconv from command line instead.
202 try:
203 call = subprocess.Popen(['iconv', '-f', 'UTF-8', '-t', charset],
204 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
205 stderr=subprocess.DEVNULL)
206 if call.poll() is not None:
207 (_, error) = call.communicate(input='')
208 print('Error: `iconv` ended with error "{}".\n'.format(error))
209 exit(1)
210 (codepoint, _) = call.communicate(input=char.encode('UTF-8'))
211 except FileNotFoundError:
212 print('Error: "{}" is not a supported charset by python and `iconv` is not installed.\n')
213 exit(1)
214
215 if codepoint == b'':
216 continue
217 # ord() is said to return the unicode codepoint.
218 # But it turns out it also gives the codepoint for other
219 # charsets if I turn the string to encoded bytes first.
220 # Not sure if that is a bug or expected.
221 codepoint = ord(codepoint)
222 if charsets[charset].charmap[codepoint] == LET:
223 characters[ord(char)] = 1
224 is_letter = True
225 break
226 if is_letter:
227 if prev_char is not None:
228 if (prev_char, ord(char)) in sequences:
229 sequences[(prev_char, ord(char))] += 1
230 else:
231 sequences[(prev_char, ord(char))] = 1
232 prev_char = ord(char)
233 else:
234 prev_char = None
235
236 def visit_pages(titles, depth, lang, logfd):
237 global visited_pages
238 global options
239
240 if len(titles) == 0:
241 return
242
243 next_titles = []
244 if options.max_page is not None:
245 max_titles = int(options.max_page/(options.max_depth * options.max_depth))
246 else:
247 max_titles = sys.maxsize
248 for title in titles:
249 if options.max_page is not None and \
250 len(visited_pages) > options.max_page:
251 return
252 if title in visited_pages:
253 continue
254
255 # Ugly hack skipping internal pages
256 if 'wiki' in title or 'Wiki' in title:
257 print('Skipping', title)
258 continue
259
260 visited_pages += [title]
261 try:
262 page = wikipedia.page(title)
263 except (wikipedia.exceptions.PageError,
264 wikipedia.exceptions.DisambiguationError):
265 # Let's just discard a page when I get an exception.
266 print("Discarding page {}.\n".format(title))
267 continue
268 logfd.write("\n{} (revision {})".format(title, page.revision_id))
269 logfd.flush()
270
271 process_text(page.content, lang)
272 try:
273 links = page.links
274 random.shuffle(links)
275 if len(links) > max_titles:
276 links = links[:max_titles]
277 next_titles += links
278 except KeyError:
279 pass
280
281 if depth >= options.max_depth:
282 return
283
284 random.shuffle(next_titles)
285 visit_pages (next_titles, depth + 1, lang, logfd)
286
287 language_c = lang.name.replace('-', '_').title()
288 build_log = current_dir + '/BuildLangModelLogs/Lang{}Model.log'.format(language_c)
289 logfd = open(build_log, 'w')
290 logfd.write('= Logs of language model for {} ({}) =\n'.format(lang.name, lang.code))
291 logfd.write('\n- Generated by {}'.format(os.path.basename(__file__)))
292 logfd.write('\n- Started: {}'.format(str(datetime.datetime.now())))
293 logfd.write('\n- Maximum depth: {}'.format(options.max_depth))
294 if options.max_page is not None:
295 logfd.write('\n- Max number of pages: {}'.format(options.max_page))
296 logfd.write('\n\n== Parsed pages ==\n')
297 logfd.flush()
298 try:
299 visit_pages(lang.start_pages, 0, lang, logfd)
300 except requests.exceptions.ConnectionError:
301 print('Error: connection to Wikipedia failed. Aborting\n')
302 exit(1)
303 logfd.write('\n\n== End of Parsed pages ==')
304 logfd.write('\n\n- Wikipedia parsing ended at: {}\n'.format(str(datetime.datetime.now())))
305 logfd.flush()
306
307 ########### CHARACTERS ###########
308
309 # Character ratios.
310 ratios = {}
311 n_char = len(characters)
312 occurrences = sum(characters.values())
313
314 logfd.write("\n{} characters appeared {} times.\n".format(n_char, occurrences))
315 for char in characters:
316 ratios[char] = characters[char] / occurrences
317 #logfd.write("Character '{}' usage: {} ({} %)\n".format(chr(char),
318 # characters[char],
319 # ratios[char] * 100))
320
321 sorted_ratios = sorted(ratios.items(), key=operator.itemgetter(1),
322 reverse=True)
323 # Accumulated ratios of the frequent chars.
324 accumulated_ratios = 0
325
326 # If there is no alphabet defined, we just use the first 64 letters, which was
327 # the original default.
328 # If there is an alphabet, we make sure all the alphabet characters are in the
329 # frequent list, and we stop then. There may therefore be more or less than
330 # 64 frequent characters depending on the language.
331 if lang.alphabet is None:
332 freq_count = 64
333 else:
334 freq_count = 0
335 for order, (char, ratio) in enumerate(sorted_ratios):
336 if len(lang.alphabet) == 0:
337 break
338 if chr(char) in lang.alphabet:
339 lang.alphabet.remove(chr(char))
340 freq_count += 1
341 else:
342 if len(lang.alphabet) > 0:
343 print("Error: alphabet characters are absent from data collection"
344 "\n Please check the configuration or the data."
345 "\n Missing characters: {}".format(", ".join(lang.alphabet)))
346 exit(1)
347
348 logfd.write('\nFirst {} characters:'.format(freq_count))
349 for order, (char, ratio) in enumerate(sorted_ratios):
350 if order >= freq_count:
351 break
352 logfd.write("\n[{:2}] Char {}: {} %".format(order, chr(char), ratio * 100))
353 accumulated_ratios += ratio
354
355 logfd.write("\n\nThe first {} characters have an accumulated ratio of {}.\n".format(freq_count, accumulated_ratios))
356
357 with open(current_dir + '/header-template.cpp', 'r') as header_fd:
358 c_code = header_fd.read()
359
360 c_code += '\n/********* Language model for: {} *********/\n\n'.format(lang.name)
361 c_code += '/**\n * Generated by {}\n'.format(os.path.basename(__file__))
362 c_code += ' * On: {}\n'.format(str(datetime.datetime.now()))
363 c_code += ' **/\n'
364
365 c_code += \
366 """
367 /* Character Mapping Table:
368 * ILL: illegal character.
369 * CTR: control character specific to the charset.
370 * RET: carriage/return.
371 * SYM: symbol (punctuation) that does not belong to word.
372 * NUM: 0 - 9.
373 *
374 * Other characters are ordered by probabilities
375 * (0 is the most common character in the language).
376 *
377 * Orders are generic to a language. So the codepoint with order X in
378 * CHARSET1 maps to the same character as the codepoint with the same
379 * order X in CHARSET2 for the same language.
380 * As such, it is possible to get missing order. For instance the
381 * ligature of 'o' and 'e' exists in ISO-8859-15 but not in ISO-8859-1
382 * even though they are both used for French. Same for the euro sign.
383 */
384 """
385
386 for charset in charsets:
387 charset_c = charset.replace('-', '_').title()
388 CTOM_str = 'static const unsigned char {}_CharToOrderMap[]'.format(charset_c)
389 CTOM_str += ' =\n{'
390 for line in range(0, 16):
391 CTOM_str += '\n '
392 for column in range(0, 16):
393 cp = line * 16 + column
394 cp_type = charsets[charset].charmap[cp]
395 if cp_type == ILL:
396 CTOM_str += 'ILL,'
397 elif cp_type == RET:
398 CTOM_str += 'RET,'
399 elif cp_type == CTR:
400 CTOM_str += 'CTR,'
401 elif cp_type == SYM:
402 CTOM_str += 'SYM,'
403 elif cp_type == NUM:
404 CTOM_str += 'NUM,'
405 else: # LET
406 try:
407 uchar = bytes([cp]).decode(charset)
408 except UnicodeDecodeError:
409 print('Unknown character 0X{:X} in {}.'.format(cp, charset))
410 print('Please verify your charset specification.\n')
411 exit(1)
412 except LookupError:
413 # Unknown encoding. Use iconv instead.
414 try:
415 call = subprocess.Popen(['iconv', '-t', 'UTF-8', '-f', charset],
416 stdin=subprocess.PIPE,
417 stdout=subprocess.PIPE,
418 stderr=subprocess.PIPE)
419 if call.poll() is not None:
420 (_, error) = call.communicate(input='')
421 print('Error: `iconv` ended with error "{}".\n'.format(error))
422 exit(1)
423 (uchar, _) = call.communicate(input=bytes([cp]))
424 uchar = uchar.decode('UTF-8')
425 except FileNotFoundError:
426 print('Error: "{}" is not a supported charset by python and `iconv` is not installed.\n')
427 exit(1)
428 #if lang.case_mapping and uchar.isupper() and \
429 #len(unicodedata.normalize('NFC', uchar.lower())) == 1:
430 # Unless we encounter special cases of characters with no
431 # composed lowercase, we lowercase it.
432 if lang.case_mapping or lang.custom_case_mapping is not None:
433 uchar = local_lowercase(uchar, lang)
434 for order, (char, ratio) in enumerate(sorted_ratios):
435 if char == ord(uchar):
436 CTOM_str += '{:3},'.format(min(249, order))
437 break
438 else:
439 # XXX: we must make sure the character order does not go
440 # over the special characters (250 currently). This may
441 # actually happen when building a model for a language
442 # writable with many different encoding. So let's just
443 # ceil the order value at 249 max.
444 # It may be an interesting alternative to add another
445 # constant for any character with an order > freqCharCount.
446 # Maybe IRR (irrelevant character) or simply CHR.
447 CTOM_str += '{:3},'.format(min(249, n_char))
448 n_char += 1
449 CTOM_str += ' /* {:X}X */'.format(line)
450 CTOM_str += '\n};\n/*'
451 CTOM_str += 'X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 XA XB XC XD XE XF'
452 CTOM_str += ' */\n\n'
453 c_code += CTOM_str
454
455 ########### SEQUENCES ###########
456
457 ratios = {}
458 occurrences = sum(sequences.values())
459 ratio_512 = 0
460 ratio_1024 = 0
461
462 sorted_seqs = sorted(sequences.items(), key=operator.itemgetter(1),
463 reverse=True)
464 for order, ((c1, c2), count) in enumerate(sorted_seqs):
465 if order < 512:
466 ratio_512 += count
467 elif order < 1024:
468 ratio_1024 += count
469 else:
470 break
471 ratio_512 /= occurrences
472 ratio_1024 /= occurrences
473
474 logfd.write("\n{} sequences found.\n".format(len(sorted_seqs)))
475
476 c_code += """
477 /* Model Table:
478 * Total sequences: {}
479 * First 512 sequences: {}
480 * Next 512 sequences (512-1024): {}
481 * Rest: {}
482 * Negative sequences: TODO""".format(len(sorted_seqs),
483 ratio_512,
484 ratio_1024,
485 1 - ratio_512 - ratio_1024)
486
487 logfd.write("\nFirst 512 (typical positive ratio): {}".format(ratio_512))
488 logfd.write("\nNext 512 (512-1024): {}".format(ratio))
489 logfd.write("\nRest: {}".format(1 - ratio_512 - ratio_1024))
490
491 c_code += "\n */\n"
492
493 LM_str = 'static const PRUint8 {}LangModel[]'.format(language_c)
494 LM_str += ' =\n{'
495 for line in range(0, freq_count):
496 LM_str += '\n '
497 for column in range(0, freq_count):
498 # Let's not make too long lines.
499 if freq_count > 40 and column == int(freq_count / 2):
500 LM_str += '\n '
501 first_order = int(line)
502 second_order = column
503 if first_order < len(sorted_ratios) and second_order < len(sorted_ratios):
504 (first_char, _) = sorted_ratios[first_order]
505 (second_char, _) = sorted_ratios[second_order]
506 if (first_char, second_char) in sequences:
507 for order, (seq, _) in enumerate(sorted_seqs):
508 if seq == (first_char, second_char):
509 if order < 512:
510 LM_str += '3,'
511 elif order < 1024:
512 LM_str += '2,'
513 else:
514 LM_str += '1,'
515 break
516 else:
517 pass # impossible!
518 LM_str += '0,'
519 else:
520 LM_str += '0,'
521 else:
522 # It may indeed happen that we find less than 64 letters used for a
523 # given language.
524 LM_str += '0,'
525 LM_str += '\n};\n'
526 c_code += LM_str
527
528 for charset in charsets:
529 charset_c = charset.replace('-', '_').title()
530 SM_str = '\n\nconst SequenceModel {}{}Model ='.format(charset_c, language_c)
531 SM_str += '\n{\n '
532 SM_str += '{}_CharToOrderMap,\n {}LangModel,'.format(charset_c, language_c)
533 SM_str += '\n {},'.format(freq_count)
534 SM_str += '\n (float){},'.format(ratio_512)
535 SM_str += '\n {},'.format('PR_TRUE' if lang.use_ascii else 'PR_FALSE')
536 SM_str += '\n "{}"'.format(charset)
537 SM_str += '\n};'
538 c_code += SM_str
539
540 c_code += '\n'
541
542 lang_model_file = current_dir + '/../src/LangModels/Lang{}Model.cpp'.format(language_c)
543 with open(lang_model_file, 'w') as cpp_fd:
544 cpp_fd.write(c_code)
545
546 logfd.write('\n\n- Processing end: {}\n'.format(str(datetime.datetime.now())))
547 logfd.close()
548
549 print("The following language model file has been generated: {}"
550 "\nThe build log is available in: {}"
551 "\nTest them and commit them.".format(lang_model_file, build_log))