| 1 |
#!/usr/bin/env python3 |
| 2 |
# -*- coding: utf-8 -*- |
| 3 |
# Usage: |
| 4 |
# convert.py input_filename |
| 5 |
# input_filename is a file of Wikipedia article titles, one title per line. |
| 6 |
|
| 7 |
import logging |
| 8 |
import regex |
| 9 |
import sys |
| 10 |
|
| 11 |
import opencc |
| 12 |
from pypinyin import lazy_pinyin |
| 13 |
|
| 14 |
# Require at least 2 characters |
| 15 |
_MINIMUM_LEN = 2 |
| 16 |
_LIST_PAGE_ENDINGS = [ |
| 17 |
'列表', |
| 18 |
'对照表', |
| 19 |
] |
| 20 |
_LOG_EVERY = 1000 |
| 21 |
|
| 22 |
_PINYIN_SEPARATOR = '\'' |
| 23 |
# https://ayaka.shn.hk/hanregex/ |
| 24 |
# INTERPUNCT \u00b7 -> · |
| 25 |
# HYPHEN-MINUS \u002d -> - |
| 26 |
# HYPHEN \u2010 -> ‐ |
| 27 |
# EN DASH \u2013 -> – |
| 28 |
# EM DASH \u2014 -> — |
| 29 |
_HANZI_RE = regex.compile(r"([\p{Unified_Ideograph}\u3006\u3007\u00b7\u002d\u2010\u2013\u2014][\ufe00-\ufe0f\U000e0100-\U000e01ef]?)+") |
| 30 |
_INTERPUNCT_TRANSTAB = str.maketrans("", "", "·-‐–—") |
| 31 |
_TO_SIMPLIFIED_CHINESE = opencc.OpenCC('t2s.json') |
| 32 |
|
| 33 |
_PINYIN_FIXES = { |
| 34 |
'n': 'en', # https://github.com/felixonmars/fcitx5-pinyin-zhwiki/issues/13 |
| 35 |
} |
| 36 |
|
| 37 |
logging.basicConfig(level=logging.INFO) |
| 38 |
|
| 39 |
|
| 40 |
def is_good_title(title, previous_title=None): |
| 41 |
if not _HANZI_RE.fullmatch(title): |
| 42 |
return False |
| 43 |
|
| 44 |
# Skip single character & too long pages |
| 45 |
if len(title) < _MINIMUM_LEN: |
| 46 |
return False |
| 47 |
|
| 48 |
# Skip list pages |
| 49 |
if title.endswith(tuple(_LIST_PAGE_ENDINGS)): |
| 50 |
return False |
| 51 |
|
| 52 |
if previous_title and \ |
| 53 |
len(previous_title) >= 4 and \ |
| 54 |
title.startswith(previous_title): |
| 55 |
return False |
| 56 |
|
| 57 |
return True |
| 58 |
|
| 59 |
|
| 60 |
def log_count(count): |
| 61 |
logging.info(f'{count} words generated') |
| 62 |
|
| 63 |
|
| 64 |
def make_output(word, pinyin): |
| 65 |
return '\t'.join([word, pinyin, '0']) |
| 66 |
|
| 67 |
|
| 68 |
def main(): |
| 69 |
previous_title = None |
| 70 |
result_count = 0 |
| 71 |
with open(sys.argv[1]) as f: |
| 72 |
for line in f: |
| 73 |
title = _TO_SIMPLIFIED_CHINESE.convert(line.strip()) |
| 74 |
if is_good_title(title, previous_title): |
| 75 |
stripped_title = title.translate(_INTERPUNCT_TRANSTAB) |
| 76 |
pinyin = [_PINYIN_FIXES.get(item, item) for item in lazy_pinyin(stripped_title)] |
| 77 |
pinyin = _PINYIN_SEPARATOR.join(pinyin) |
| 78 |
if _HANZI_RE.search(pinyin): |
| 79 |
logging.info( |
| 80 |
f'Failed to convert to Pinyin. Ignoring: {pinyin}') |
| 81 |
continue |
| 82 |
print(make_output(title, pinyin)) |
| 83 |
result_count += 1 |
| 84 |
if result_count % _LOG_EVERY == 0: |
| 85 |
log_count(result_count) |
| 86 |
previous_title = title |
| 87 |
log_count(result_count) |
| 88 |
|
| 89 |
|
| 90 |
if __name__ == '__main__': |
| 91 |
main() |