| 1 |
#!/bin/python3 |
| 2 |
# -*- coding: utf-8 -*- |
| 3 |
|
| 4 |
# ##### BEGIN LICENSE BLOCK ##### |
| 5 |
# Version: MPL 1.1/GPL 2.0/LGPL 2.1 |
| 6 |
# |
| 7 |
# The contents of this file are subject to the Mozilla Public License Version |
| 8 |
# 1.1 (the "License"); you may not use this file except in compliance with |
| 9 |
# the License. You may obtain a copy of the License at |
| 10 |
# http://www.mozilla.org/MPL/ |
| 11 |
# |
| 12 |
# Software distributed under the License is distributed on an "AS IS" basis, |
| 13 |
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License |
| 14 |
# for the specific language governing rights and limitations under the |
| 15 |
# License. |
| 16 |
# |
| 17 |
# The Original Code is Mozilla Universal charset detector code. |
| 18 |
# |
| 19 |
# The Initial Developer of the Original Code is |
| 20 |
# Netscape Communications Corporation. |
| 21 |
# Portions created by the Initial Developer are Copyright (C) 2001 |
| 22 |
# the Initial Developer. All Rights Reserved. |
| 23 |
# |
| 24 |
# Contributor(s): |
| 25 |
# Jehan <[email protected]> |
| 26 |
# |
| 27 |
# Alternatively, the contents of this file may be used under the terms of |
| 28 |
# either the GNU General Public License Version 2 or later (the "GPL"), or |
| 29 |
# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), |
| 30 |
# in which case the provisions of the GPL or the LGPL are applicable instead |
| 31 |
# of those above. If you wish to allow use of your version of this file only |
| 32 |
# under the terms of either the GPL or the LGPL, and not to allow others to |
| 33 |
# use your version of this file under the terms of the MPL, indicate your |
| 34 |
# decision by deleting the provisions above and replace them with the notice |
| 35 |
# and other provisions required by the GPL or the LGPL. If you do not delete |
| 36 |
# the provisions above, a recipient may use your version of this file under |
| 37 |
# the terms of any one of the MPL, the GPL or the LGPL. |
| 38 |
# |
| 39 |
# ##### END LICENSE BLOCK ##### |
| 40 |
|
| 41 |
# Third party modules. |
| 42 |
import unicodedata |
| 43 |
import subprocess |
| 44 |
import wikipedia |
| 45 |
import importlib |
| 46 |
import optparse |
| 47 |
import datetime |
| 48 |
import operator |
| 49 |
import requests |
| 50 |
import sys |
| 51 |
import re |
| 52 |
import os |
| 53 |
import random |
| 54 |
|
| 55 |
# Custom modules. |
| 56 |
import charsets.db |
| 57 |
from charsets.codepoints import * |
| 58 |
|
| 59 |
# Command line processing. |
| 60 |
usage = 'Usage: {} <LANG-CODE>\n' \ |
| 61 |
'\nEx: `{} fr`'.format(__file__, __file__) |
| 62 |
|
| 63 |
description = "Internal tool for uchardet to generate language data." |
| 64 |
cmdline = optparse.OptionParser(usage, description = description) |
| 65 |
cmdline.add_option('--max-page', |
| 66 |
help = 'Maximum number of Wikipedia pages to parse (useful for debugging).', |
| 67 |
action = 'store', type = 'int', dest = 'max_page', default = None) |
| 68 |
cmdline.add_option('--max-depth', |
| 69 |
help = 'Maximum depth when following links from start page (default: 2).', |
| 70 |
action = 'store', type = 'int', |
| 71 |
dest = 'max_depth', default = 2) |
| 72 |
(options, langs) = cmdline.parse_args() |
| 73 |
if len(langs) < 1: |
| 74 |
print("Please select at least one language code.\n") |
| 75 |
exit(1) |
| 76 |
if len(langs) > 1: |
| 77 |
print("This script is meant to generate data for one language at a time.\n") |
| 78 |
exit(1) |
| 79 |
lang = langs[0] |
| 80 |
|
| 81 |
# Load the language data. |
| 82 |
sys_path_backup = sys.path |
| 83 |
current_dir = os.path.dirname(os.path.realpath(__file__)) |
| 84 |
sys.path = [current_dir + '/langs'] |
| 85 |
|
| 86 |
try: |
| 87 |
lang = importlib.import_module(lang.lower()) |
| 88 |
except ImportError: |
| 89 |
print('Unknown language code "{}": ' |
| 90 |
'file "langs/{}.py" does not exist.'.format(lang, lang.lower())) |
| 91 |
exit(1) |
| 92 |
sys.path = sys_path_backup |
| 93 |
|
| 94 |
charsets = charsets.db.load(lang.charsets) |
| 95 |
|
| 96 |
if not hasattr(lang, 'start_pages') or lang.start_pages is None or \ |
| 97 |
lang.start_pages == []: |
| 98 |
# Let's start with the main page, assuming it should have links |
| 99 |
# to relevant pages. In locale wikipedia, this page is usually redirected |
| 100 |
# to a relevant page. |
| 101 |
print("Warning: no `start_pages` set for '{}'. Using ['Main_Page'].\n" |
| 102 |
" If you don't get good data, it is advised to set a " |
| 103 |
"start_pages` variable yourself.".format(lang.code)) |
| 104 |
lang.start_pages = ['Main_Page'] |
| 105 |
if not hasattr(lang, 'wikipedia_code') or lang.wikipedia_code is None: |
| 106 |
lang.wikipedia_code = lang.code |
| 107 |
if not hasattr(lang, 'clean_wikipedia_content') or lang.clean_wikipedia_content is None: |
| 108 |
lang.clean_wikipedia_content = None |
| 109 |
if hasattr(lang, 'case_mapping'): |
| 110 |
lang.case_mapping = bool(lang.case_mapping) |
| 111 |
else: |
| 112 |
lang.case_mapping = False |
| 113 |
if not hasattr(lang, 'custom_case_mapping'): |
| 114 |
lang.custom_case_mapping = None |
| 115 |
if not hasattr(lang, 'alphabet') or lang.alphabet is None: |
| 116 |
lang.alphabet = None |
| 117 |
|
| 118 |
def local_lowercase(text, lang): |
| 119 |
lowercased = '' |
| 120 |
for l in text: |
| 121 |
if lang.custom_case_mapping is not None and \ |
| 122 |
l in lang.custom_case_mapping: |
| 123 |
lowercased += lang.custom_case_mapping[l] |
| 124 |
elif l.isupper() and \ |
| 125 |
lang.case_mapping and \ |
| 126 |
len(unicodedata.normalize('NFC', l.lower())) == 1: |
| 127 |
lowercased += l.lower() |
| 128 |
else: |
| 129 |
lowercased += l |
| 130 |
return lowercased |
| 131 |
|
| 132 |
if lang.alphabet is not None: |
| 133 |
# Allowing to provide an alphabet in string format rather than list. |
| 134 |
lang.alphabet = list(lang.alphabet) |
| 135 |
if lang.use_ascii: |
| 136 |
lang.alphabet += [chr(l) for l in range(65, 91)] + [chr(l) for l in range(97, 123)] |
| 137 |
if lang.case_mapping or lang.custom_case_mapping is not None: |
| 138 |
lang.alphabet = [local_lowercase(l, lang) for l in lang.alphabet] |
| 139 |
#alphabet = [] |
| 140 |
#for l in lang.alphabet: |
| 141 |
#if l.isupper() and \ |
| 142 |
#lang.custom_case_mapping is not None and \ |
| 143 |
#l in lang.custom_case_mapping: |
| 144 |
#alphabet.append(lang.custom_case_mapping[l]) |
| 145 |
#elif l.isupper() and \ |
| 146 |
#lang.case_mapping and \ |
| 147 |
#len(unicodedata.normalize('NFC', l.lower())) == 1: |
| 148 |
#alphabet.append(l.lower()) |
| 149 |
#else: |
| 150 |
#alphabet.append(l) |
| 151 |
lang.alphabet = list(set(lang.alphabet)) |
| 152 |
|
| 153 |
# Starting processing. |
| 154 |
wikipedia.set_lang(lang.wikipedia_code) |
| 155 |
|
| 156 |
visited_pages = [] |
| 157 |
|
| 158 |
# The full list of letter characters. |
| 159 |
# The key is the unicode codepoint, |
| 160 |
# and the value is the occurrence count. |
| 161 |
characters = {} |
| 162 |
# Sequence of letters. |
| 163 |
# The key is the couple (char1, char2) in unicode codepoint, |
| 164 |
# the value is the occurrence count. |
| 165 |
sequences = {} |
| 166 |
prev_char = None |
| 167 |
|
| 168 |
def process_text(content, lang): |
| 169 |
global charsets |
| 170 |
global characters |
| 171 |
global sequences |
| 172 |
global prev_char |
| 173 |
|
| 174 |
if lang.clean_wikipedia_content is not None: |
| 175 |
content = lang.clean_wikipedia_content(content) |
| 176 |
# Clean out the Wikipedia syntax for titles. |
| 177 |
content = re.sub(r'(=+) *([^=]+) *\1', |
| 178 |
r'\2', content) |
| 179 |
# Clean multiple spaces. Newlines and such are normalized to spaces, |
| 180 |
# since they have basically a similar role in the purpose of uchardet. |
| 181 |
content = re.sub(r'\s+', ' ', content) |
| 182 |
|
| 183 |
if lang.case_mapping or lang.custom_case_mapping is not None: |
| 184 |
content = local_lowercase(content, lang) |
| 185 |
|
| 186 |
# In python 3, strings are UTF-8. |
| 187 |
# Looping through them return expected characters. |
| 188 |
for char in content: |
| 189 |
is_letter = False |
| 190 |
if ord(char) in characters: |
| 191 |
characters[ord(char)] += 1 |
| 192 |
is_letter = True |
| 193 |
else: |
| 194 |
# We save the character if it is at least in one of the |
| 195 |
# language encodings and its not a special character. |
| 196 |
for charset in charsets: |
| 197 |
# Does the character exist in the charset? |
| 198 |
try: |
| 199 |
codepoint = char.encode(charset, 'ignore') |
| 200 |
except LookupError: |
| 201 |
# unknown encoding. Use iconv from command line instead. |
| 202 |
try: |
| 203 |
call = subprocess.Popen(['iconv', '-f', 'UTF-8', '-t', charset], |
| 204 |
stdin=subprocess.PIPE, stdout=subprocess.PIPE, |
| 205 |
stderr=subprocess.DEVNULL) |
| 206 |
if call.poll() is not None: |
| 207 |
(_, error) = call.communicate(input='') |
| 208 |
print('Error: `iconv` ended with error "{}".\n'.format(error)) |
| 209 |
exit(1) |
| 210 |
(codepoint, _) = call.communicate(input=char.encode('UTF-8')) |
| 211 |
except FileNotFoundError: |
| 212 |
print('Error: "{}" is not a supported charset by python and `iconv` is not installed.\n') |
| 213 |
exit(1) |
| 214 |
|
| 215 |
if codepoint == b'': |
| 216 |
continue |
| 217 |
# ord() is said to return the unicode codepoint. |
| 218 |
# But it turns out it also gives the codepoint for other |
| 219 |
# charsets if I turn the string to encoded bytes first. |
| 220 |
# Not sure if that is a bug or expected. |
| 221 |
codepoint = ord(codepoint) |
| 222 |
if charsets[charset].charmap[codepoint] == LET: |
| 223 |
characters[ord(char)] = 1 |
| 224 |
is_letter = True |
| 225 |
break |
| 226 |
if is_letter: |
| 227 |
if prev_char is not None: |
| 228 |
if (prev_char, ord(char)) in sequences: |
| 229 |
sequences[(prev_char, ord(char))] += 1 |
| 230 |
else: |
| 231 |
sequences[(prev_char, ord(char))] = 1 |
| 232 |
prev_char = ord(char) |
| 233 |
else: |
| 234 |
prev_char = None |
| 235 |
|
| 236 |
def visit_pages(titles, depth, lang, logfd): |
| 237 |
global visited_pages |
| 238 |
global options |
| 239 |
|
| 240 |
if len(titles) == 0: |
| 241 |
return |
| 242 |
|
| 243 |
next_titles = [] |
| 244 |
if options.max_page is not None: |
| 245 |
max_titles = int(options.max_page/(options.max_depth * options.max_depth)) |
| 246 |
else: |
| 247 |
max_titles = sys.maxsize |
| 248 |
for title in titles: |
| 249 |
if options.max_page is not None and \ |
| 250 |
len(visited_pages) > options.max_page: |
| 251 |
return |
| 252 |
if title in visited_pages: |
| 253 |
continue |
| 254 |
|
| 255 |
# Ugly hack skipping internal pages |
| 256 |
if 'wiki' in title or 'Wiki' in title: |
| 257 |
print('Skipping', title) |
| 258 |
continue |
| 259 |
|
| 260 |
visited_pages += [title] |
| 261 |
try: |
| 262 |
page = wikipedia.page(title) |
| 263 |
except (wikipedia.exceptions.PageError, |
| 264 |
wikipedia.exceptions.DisambiguationError): |
| 265 |
# Let's just discard a page when I get an exception. |
| 266 |
print("Discarding page {}.\n".format(title)) |
| 267 |
continue |
| 268 |
logfd.write("\n{} (revision {})".format(title, page.revision_id)) |
| 269 |
logfd.flush() |
| 270 |
|
| 271 |
process_text(page.content, lang) |
| 272 |
try: |
| 273 |
links = page.links |
| 274 |
random.shuffle(links) |
| 275 |
if len(links) > max_titles: |
| 276 |
links = links[:max_titles] |
| 277 |
next_titles += links |
| 278 |
except KeyError: |
| 279 |
pass |
| 280 |
|
| 281 |
if depth >= options.max_depth: |
| 282 |
return |
| 283 |
|
| 284 |
random.shuffle(next_titles) |
| 285 |
visit_pages (next_titles, depth + 1, lang, logfd) |
| 286 |
|
| 287 |
language_c = lang.name.replace('-', '_').title() |
| 288 |
build_log = current_dir + '/BuildLangModelLogs/Lang{}Model.log'.format(language_c) |
| 289 |
logfd = open(build_log, 'w') |
| 290 |
logfd.write('= Logs of language model for {} ({}) =\n'.format(lang.name, lang.code)) |
| 291 |
logfd.write('\n- Generated by {}'.format(os.path.basename(__file__))) |
| 292 |
logfd.write('\n- Started: {}'.format(str(datetime.datetime.now()))) |
| 293 |
logfd.write('\n- Maximum depth: {}'.format(options.max_depth)) |
| 294 |
if options.max_page is not None: |
| 295 |
logfd.write('\n- Max number of pages: {}'.format(options.max_page)) |
| 296 |
logfd.write('\n\n== Parsed pages ==\n') |
| 297 |
logfd.flush() |
| 298 |
try: |
| 299 |
visit_pages(lang.start_pages, 0, lang, logfd) |
| 300 |
except requests.exceptions.ConnectionError: |
| 301 |
print('Error: connection to Wikipedia failed. Aborting\n') |
| 302 |
exit(1) |
| 303 |
logfd.write('\n\n== End of Parsed pages ==') |
| 304 |
logfd.write('\n\n- Wikipedia parsing ended at: {}\n'.format(str(datetime.datetime.now()))) |
| 305 |
logfd.flush() |
| 306 |
|
| 307 |
########### CHARACTERS ########### |
| 308 |
|
| 309 |
# Character ratios. |
| 310 |
ratios = {} |
| 311 |
n_char = len(characters) |
| 312 |
occurrences = sum(characters.values()) |
| 313 |
|
| 314 |
logfd.write("\n{} characters appeared {} times.\n".format(n_char, occurrences)) |
| 315 |
for char in characters: |
| 316 |
ratios[char] = characters[char] / occurrences |
| 317 |
#logfd.write("Character '{}' usage: {} ({} %)\n".format(chr(char), |
| 318 |
# characters[char], |
| 319 |
# ratios[char] * 100)) |
| 320 |
|
| 321 |
sorted_ratios = sorted(ratios.items(), key=operator.itemgetter(1), |
| 322 |
reverse=True) |
| 323 |
# Accumulated ratios of the frequent chars. |
| 324 |
accumulated_ratios = 0 |
| 325 |
|
| 326 |
# If there is no alphabet defined, we just use the first 64 letters, which was |
| 327 |
# the original default. |
| 328 |
# If there is an alphabet, we make sure all the alphabet characters are in the |
| 329 |
# frequent list, and we stop then. There may therefore be more or less than |
| 330 |
# 64 frequent characters depending on the language. |
| 331 |
if lang.alphabet is None: |
| 332 |
freq_count = 64 |
| 333 |
else: |
| 334 |
freq_count = 0 |
| 335 |
for order, (char, ratio) in enumerate(sorted_ratios): |
| 336 |
if len(lang.alphabet) == 0: |
| 337 |
break |
| 338 |
if chr(char) in lang.alphabet: |
| 339 |
lang.alphabet.remove(chr(char)) |
| 340 |
freq_count += 1 |
| 341 |
else: |
| 342 |
if len(lang.alphabet) > 0: |
| 343 |
print("Error: alphabet characters are absent from data collection" |
| 344 |
"\n Please check the configuration or the data." |
| 345 |
"\n Missing characters: {}".format(", ".join(lang.alphabet))) |
| 346 |
exit(1) |
| 347 |
|
| 348 |
logfd.write('\nFirst {} characters:'.format(freq_count)) |
| 349 |
for order, (char, ratio) in enumerate(sorted_ratios): |
| 350 |
if order >= freq_count: |
| 351 |
break |
| 352 |
logfd.write("\n[{:2}] Char {}: {} %".format(order, chr(char), ratio * 100)) |
| 353 |
accumulated_ratios += ratio |
| 354 |
|
| 355 |
logfd.write("\n\nThe first {} characters have an accumulated ratio of {}.\n".format(freq_count, accumulated_ratios)) |
| 356 |
|
| 357 |
with open(current_dir + '/header-template.cpp', 'r') as header_fd: |
| 358 |
c_code = header_fd.read() |
| 359 |
|
| 360 |
c_code += '\n/********* Language model for: {} *********/\n\n'.format(lang.name) |
| 361 |
c_code += '/**\n * Generated by {}\n'.format(os.path.basename(__file__)) |
| 362 |
c_code += ' * On: {}\n'.format(str(datetime.datetime.now())) |
| 363 |
c_code += ' **/\n' |
| 364 |
|
| 365 |
c_code += \ |
| 366 |
""" |
| 367 |
/* Character Mapping Table: |
| 368 |
* ILL: illegal character. |
| 369 |
* CTR: control character specific to the charset. |
| 370 |
* RET: carriage/return. |
| 371 |
* SYM: symbol (punctuation) that does not belong to word. |
| 372 |
* NUM: 0 - 9. |
| 373 |
* |
| 374 |
* Other characters are ordered by probabilities |
| 375 |
* (0 is the most common character in the language). |
| 376 |
* |
| 377 |
* Orders are generic to a language. So the codepoint with order X in |
| 378 |
* CHARSET1 maps to the same character as the codepoint with the same |
| 379 |
* order X in CHARSET2 for the same language. |
| 380 |
* As such, it is possible to get missing order. For instance the |
| 381 |
* ligature of 'o' and 'e' exists in ISO-8859-15 but not in ISO-8859-1 |
| 382 |
* even though they are both used for French. Same for the euro sign. |
| 383 |
*/ |
| 384 |
""" |
| 385 |
|
| 386 |
for charset in charsets: |
| 387 |
charset_c = charset.replace('-', '_').title() |
| 388 |
CTOM_str = 'static const unsigned char {}_CharToOrderMap[]'.format(charset_c) |
| 389 |
CTOM_str += ' =\n{' |
| 390 |
for line in range(0, 16): |
| 391 |
CTOM_str += '\n ' |
| 392 |
for column in range(0, 16): |
| 393 |
cp = line * 16 + column |
| 394 |
cp_type = charsets[charset].charmap[cp] |
| 395 |
if cp_type == ILL: |
| 396 |
CTOM_str += 'ILL,' |
| 397 |
elif cp_type == RET: |
| 398 |
CTOM_str += 'RET,' |
| 399 |
elif cp_type == CTR: |
| 400 |
CTOM_str += 'CTR,' |
| 401 |
elif cp_type == SYM: |
| 402 |
CTOM_str += 'SYM,' |
| 403 |
elif cp_type == NUM: |
| 404 |
CTOM_str += 'NUM,' |
| 405 |
else: # LET |
| 406 |
try: |
| 407 |
uchar = bytes([cp]).decode(charset) |
| 408 |
except UnicodeDecodeError: |
| 409 |
print('Unknown character 0X{:X} in {}.'.format(cp, charset)) |
| 410 |
print('Please verify your charset specification.\n') |
| 411 |
exit(1) |
| 412 |
except LookupError: |
| 413 |
# Unknown encoding. Use iconv instead. |
| 414 |
try: |
| 415 |
call = subprocess.Popen(['iconv', '-t', 'UTF-8', '-f', charset], |
| 416 |
stdin=subprocess.PIPE, |
| 417 |
stdout=subprocess.PIPE, |
| 418 |
stderr=subprocess.PIPE) |
| 419 |
if call.poll() is not None: |
| 420 |
(_, error) = call.communicate(input='') |
| 421 |
print('Error: `iconv` ended with error "{}".\n'.format(error)) |
| 422 |
exit(1) |
| 423 |
(uchar, _) = call.communicate(input=bytes([cp])) |
| 424 |
uchar = uchar.decode('UTF-8') |
| 425 |
except FileNotFoundError: |
| 426 |
print('Error: "{}" is not a supported charset by python and `iconv` is not installed.\n') |
| 427 |
exit(1) |
| 428 |
#if lang.case_mapping and uchar.isupper() and \ |
| 429 |
#len(unicodedata.normalize('NFC', uchar.lower())) == 1: |
| 430 |
# Unless we encounter special cases of characters with no |
| 431 |
# composed lowercase, we lowercase it. |
| 432 |
if lang.case_mapping or lang.custom_case_mapping is not None: |
| 433 |
uchar = local_lowercase(uchar, lang) |
| 434 |
for order, (char, ratio) in enumerate(sorted_ratios): |
| 435 |
if char == ord(uchar): |
| 436 |
CTOM_str += '{:3},'.format(min(249, order)) |
| 437 |
break |
| 438 |
else: |
| 439 |
# XXX: we must make sure the character order does not go |
| 440 |
# over the special characters (250 currently). This may |
| 441 |
# actually happen when building a model for a language |
| 442 |
# writable with many different encoding. So let's just |
| 443 |
# ceil the order value at 249 max. |
| 444 |
# It may be an interesting alternative to add another |
| 445 |
# constant for any character with an order > freqCharCount. |
| 446 |
# Maybe IRR (irrelevant character) or simply CHR. |
| 447 |
CTOM_str += '{:3},'.format(min(249, n_char)) |
| 448 |
n_char += 1 |
| 449 |
CTOM_str += ' /* {:X}X */'.format(line) |
| 450 |
CTOM_str += '\n};\n/*' |
| 451 |
CTOM_str += 'X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 XA XB XC XD XE XF' |
| 452 |
CTOM_str += ' */\n\n' |
| 453 |
c_code += CTOM_str |
| 454 |
|
| 455 |
########### SEQUENCES ########### |
| 456 |
|
| 457 |
ratios = {} |
| 458 |
occurrences = sum(sequences.values()) |
| 459 |
ratio_512 = 0 |
| 460 |
ratio_1024 = 0 |
| 461 |
|
| 462 |
sorted_seqs = sorted(sequences.items(), key=operator.itemgetter(1), |
| 463 |
reverse=True) |
| 464 |
for order, ((c1, c2), count) in enumerate(sorted_seqs): |
| 465 |
if order < 512: |
| 466 |
ratio_512 += count |
| 467 |
elif order < 1024: |
| 468 |
ratio_1024 += count |
| 469 |
else: |
| 470 |
break |
| 471 |
ratio_512 /= occurrences |
| 472 |
ratio_1024 /= occurrences |
| 473 |
|
| 474 |
logfd.write("\n{} sequences found.\n".format(len(sorted_seqs))) |
| 475 |
|
| 476 |
c_code += """ |
| 477 |
/* Model Table: |
| 478 |
* Total sequences: {} |
| 479 |
* First 512 sequences: {} |
| 480 |
* Next 512 sequences (512-1024): {} |
| 481 |
* Rest: {} |
| 482 |
* Negative sequences: TODO""".format(len(sorted_seqs), |
| 483 |
ratio_512, |
| 484 |
ratio_1024, |
| 485 |
1 - ratio_512 - ratio_1024) |
| 486 |
|
| 487 |
logfd.write("\nFirst 512 (typical positive ratio): {}".format(ratio_512)) |
| 488 |
logfd.write("\nNext 512 (512-1024): {}".format(ratio)) |
| 489 |
logfd.write("\nRest: {}".format(1 - ratio_512 - ratio_1024)) |
| 490 |
|
| 491 |
c_code += "\n */\n" |
| 492 |
|
| 493 |
LM_str = 'static const PRUint8 {}LangModel[]'.format(language_c) |
| 494 |
LM_str += ' =\n{' |
| 495 |
for line in range(0, freq_count): |
| 496 |
LM_str += '\n ' |
| 497 |
for column in range(0, freq_count): |
| 498 |
# Let's not make too long lines. |
| 499 |
if freq_count > 40 and column == int(freq_count / 2): |
| 500 |
LM_str += '\n ' |
| 501 |
first_order = int(line) |
| 502 |
second_order = column |
| 503 |
if first_order < len(sorted_ratios) and second_order < len(sorted_ratios): |
| 504 |
(first_char, _) = sorted_ratios[first_order] |
| 505 |
(second_char, _) = sorted_ratios[second_order] |
| 506 |
if (first_char, second_char) in sequences: |
| 507 |
for order, (seq, _) in enumerate(sorted_seqs): |
| 508 |
if seq == (first_char, second_char): |
| 509 |
if order < 512: |
| 510 |
LM_str += '3,' |
| 511 |
elif order < 1024: |
| 512 |
LM_str += '2,' |
| 513 |
else: |
| 514 |
LM_str += '1,' |
| 515 |
break |
| 516 |
else: |
| 517 |
pass # impossible! |
| 518 |
LM_str += '0,' |
| 519 |
else: |
| 520 |
LM_str += '0,' |
| 521 |
else: |
| 522 |
# It may indeed happen that we find less than 64 letters used for a |
| 523 |
# given language. |
| 524 |
LM_str += '0,' |
| 525 |
LM_str += '\n};\n' |
| 526 |
c_code += LM_str |
| 527 |
|
| 528 |
for charset in charsets: |
| 529 |
charset_c = charset.replace('-', '_').title() |
| 530 |
SM_str = '\n\nconst SequenceModel {}{}Model ='.format(charset_c, language_c) |
| 531 |
SM_str += '\n{\n ' |
| 532 |
SM_str += '{}_CharToOrderMap,\n {}LangModel,'.format(charset_c, language_c) |
| 533 |
SM_str += '\n {},'.format(freq_count) |
| 534 |
SM_str += '\n (float){},'.format(ratio_512) |
| 535 |
SM_str += '\n {},'.format('PR_TRUE' if lang.use_ascii else 'PR_FALSE') |
| 536 |
SM_str += '\n "{}"'.format(charset) |
| 537 |
SM_str += '\n};' |
| 538 |
c_code += SM_str |
| 539 |
|
| 540 |
c_code += '\n' |
| 541 |
|
| 542 |
lang_model_file = current_dir + '/../src/LangModels/Lang{}Model.cpp'.format(language_c) |
| 543 |
with open(lang_model_file, 'w') as cpp_fd: |
| 544 |
cpp_fd.write(c_code) |
| 545 |
|
| 546 |
logfd.write('\n\n- Processing end: {}\n'.format(str(datetime.datetime.now()))) |
| 547 |
logfd.close() |
| 548 |
|
| 549 |
print("The following language model file has been generated: {}" |
| 550 |
"\nThe build log is available in: {}" |
| 551 |
"\nTest them and commit them.".format(lang_model_file, build_log)) |