| 1 |
#!/usr/bin/env python3 |
| 2 |
# -*- coding: utf-8 -*- |
| 3 |
|
| 4 |
import urllib.parse |
| 5 |
import collections |
| 6 |
import sys |
| 7 |
from mediawiki import init_session, do_request |
| 8 |
|
| 9 |
def fetch(): |
| 10 |
# https://www.mediawiki.org/wiki/API:REST_API/Reference#Get_page_source |
| 11 |
_ZHWIKI_SOURCE_URL = "https://zh.wikipedia.org/w/rest.php/v1/page/" |
| 12 |
_PAGE = "中国大陆网络用语列表" |
| 13 |
|
| 14 |
url = _ZHWIKI_SOURCE_URL + urllib.parse.quote(_PAGE) |
| 15 |
init_session() |
| 16 |
r = do_request(url) |
| 17 |
page = r.json() |
| 18 |
return page["source"] |
| 19 |
|
| 20 |
|
| 21 |
def trim_templates(wikitext): |
| 22 |
template_level = 0 |
| 23 |
new_wikitext = "" |
| 24 |
while True: |
| 25 |
assert template_level >= 0, ValueError("Unbalanced template in wikitext:\n" + wikitext) |
| 26 |
pre_open, open_tag, post_open = wikitext.partition("{{") |
| 27 |
pre_close, close_tag, post_close = wikitext.partition("}}") |
| 28 |
if open_tag and (not close_tag or len(pre_open) < len(pre_close)): |
| 29 |
# Template starts here ({{) |
| 30 |
wikitext = post_open |
| 31 |
if template_level == 0: |
| 32 |
new_wikitext += pre_open |
| 33 |
template_level += 1 |
| 34 |
elif close_tag: |
| 35 |
# Template ends here (}}) |
| 36 |
wikitext = post_close |
| 37 |
template_level -= 1 |
| 38 |
else: |
| 39 |
# No more templates |
| 40 |
assert template_level == 0, ValueError("Unbalanced template in wikitext:\n" + wikitext) |
| 41 |
# The assertion below must be true on earth |
| 42 |
assert open_tag == close_tag == "", RuntimeError("Cosmic radiation detected") |
| 43 |
new_wikitext += wikitext |
| 44 |
break |
| 45 |
|
| 46 |
return new_wikitext |
| 47 |
|
| 48 |
|
| 49 |
def process(wikitext): |
| 50 |
wikitext = trim_templates(wikitext) |
| 51 |
words = collections.OrderedDict() |
| 52 |
|
| 53 |
def add_word(word): |
| 54 |
for garbage in ("[", "]", "…", ":", ":", ")", ")", '"', "“", "”", "-{", "}-", "简称", "簡稱"): |
| 55 |
word = word.replace(garbage, "") |
| 56 |
words[word.strip()] = None |
| 57 |
|
| 58 |
def add_words(word): |
| 59 |
for word_separator in ("、", "/", "|", ",", "。", "?", "?", "(", "("): |
| 60 |
if word_separator in word: |
| 61 |
for w in word.split(word_separator): |
| 62 |
# recursively resolve |
| 63 |
add_words(w.strip()) |
| 64 |
break |
| 65 |
else: |
| 66 |
add_word(word) |
| 67 |
|
| 68 |
def iter_bolds(line): |
| 69 |
line_bak = line |
| 70 |
while "'''" in line: |
| 71 |
_, sep1, line = line.partition("'''") |
| 72 |
bold, sep2, line = line.partition("'''") |
| 73 |
assert sep1 and sep2, ValueError("Unclosed ''' in line: " + line_bak) |
| 74 |
yield bold |
| 75 |
|
| 76 |
for line in wikitext.split("\n"): |
| 77 |
if not line.startswith("*"): |
| 78 |
continue |
| 79 |
# Lists |
| 80 |
line = line.strip("*").strip() |
| 81 |
pre_colon, sep, post_colon = line.partition("''':") |
| 82 |
if not sep: |
| 83 |
pre_colon, sep, post_colon = line.partition("''':") |
| 84 |
for bold in iter_bolds(pre_colon + sep): |
| 85 |
# Add bold words before colon |
| 86 |
add_words(bold) |
| 87 |
for bold in iter_bolds(post_colon): |
| 88 |
# Add bold words after colon (or line w/o colon), skipping the origin of abbreviation (length probably <= 2) |
| 89 |
if len(bold) > 2: |
| 90 |
add_words(bold) |
| 91 |
|
| 92 |
return words |
| 93 |
|
| 94 |
|
| 95 |
def print_words(words): |
| 96 |
for word in words: |
| 97 |
print(word) |
| 98 |
|
| 99 |
|
| 100 |
if __name__ == "__main__": |
| 101 |
if len(sys.argv) == 1: |
| 102 |
wikitext = fetch() |
| 103 |
words = process(wikitext) |
| 104 |
print_words(words) |
| 105 |
|
| 106 |
elif sys.argv[1] == "--fetch": |
| 107 |
print(fetch()) |
| 108 |
|
| 109 |
elif sys.argv[1] == "--process": |
| 110 |
wikitext = open(sys.argv[2]).read() |
| 111 |
print_words(process(wikitext)) |
| 112 |
|
| 113 |
else: |
| 114 |
raise NotImplementedError |