Debian Codegraph @

convert.py

raw
1 #!/usr/bin/env python3
2 # -*- coding: utf-8 -*-
3 # Usage:
4 # convert.py input_filename
5 # input_filename is a file of Wikipedia article titles, one title per line.
6
7 import logging
8 import regex
9 import sys
10
11 import opencc
12 from pypinyin import lazy_pinyin
13
14 # Require at least 2 characters
15 _MINIMUM_LEN = 2
16 _LIST_PAGE_ENDINGS = [
17 '列表',
18 '对照表',
19 ]
20 _LOG_EVERY = 1000
21
22 _PINYIN_SEPARATOR = '\''
23 # https://ayaka.shn.hk/hanregex/
24 # INTERPUNCT \u00b7 -> ·
25 # HYPHEN-MINUS \u002d -> -
26 # HYPHEN \u2010 -> ‐
27 # EN DASH \u2013 -> –
28 # EM DASH \u2014 -> —
29 _HANZI_RE = regex.compile(r"([\p{Unified_Ideograph}\u3006\u3007\u00b7\u002d\u2010\u2013\u2014][\ufe00-\ufe0f\U000e0100-\U000e01ef]?)+")
30 _INTERPUNCT_TRANSTAB = str.maketrans("", "", "·-‐–—")
31 _TO_SIMPLIFIED_CHINESE = opencc.OpenCC('t2s.json')
32
33 _PINYIN_FIXES = {
34 'n': 'en', # https://github.com/felixonmars/fcitx5-pinyin-zhwiki/issues/13
35 }
36
37 logging.basicConfig(level=logging.INFO)
38
39
40 def is_good_title(title, previous_title=None):
41 if not _HANZI_RE.fullmatch(title):
42 return False
43
44 # Skip single character & too long pages
45 if len(title) < _MINIMUM_LEN:
46 return False
47
48 # Skip list pages
49 if title.endswith(tuple(_LIST_PAGE_ENDINGS)):
50 return False
51
52 if previous_title and \
53 len(previous_title) >= 4 and \
54 title.startswith(previous_title):
55 return False
56
57 return True
58
59
60 def log_count(count):
61 logging.info(f'{count} words generated')
62
63
64 def make_output(word, pinyin):
65 return '\t'.join([word, pinyin, '0'])
66
67
68 def main():
69 previous_title = None
70 result_count = 0
71 with open(sys.argv[1]) as f:
72 for line in f:
73 title = _TO_SIMPLIFIED_CHINESE.convert(line.strip())
74 if is_good_title(title, previous_title):
75 stripped_title = title.translate(_INTERPUNCT_TRANSTAB)
76 pinyin = [_PINYIN_FIXES.get(item, item) for item in lazy_pinyin(stripped_title)]
77 pinyin = _PINYIN_SEPARATOR.join(pinyin)
78 if _HANZI_RE.search(pinyin):
79 logging.info(
80 f'Failed to convert to Pinyin. Ignoring: {pinyin}')
81 continue
82 print(make_output(title, pinyin))
83 result_count += 1
84 if result_count % _LOG_EVERY == 0:
85 log_count(result_count)
86 previous_title = title
87 log_count(result_count)
88
89
90 if __name__ == '__main__':
91 main()