Debian Codegraph @

zhwiki-web-slang.py

raw
1 #!/usr/bin/env python3
2 # -*- coding: utf-8 -*-
3
4 import urllib.parse
5 import collections
6 import sys
7 from mediawiki import init_session, do_request
8
9 def fetch():
10 # https://www.mediawiki.org/wiki/API:REST_API/Reference#Get_page_source
11 _ZHWIKI_SOURCE_URL = "https://zh.wikipedia.org/w/rest.php/v1/page/"
12 _PAGE = "中国大陆网络用语列表"
13
14 url = _ZHWIKI_SOURCE_URL + urllib.parse.quote(_PAGE)
15 init_session()
16 r = do_request(url)
17 page = r.json()
18 return page["source"]
19
20
21 def trim_templates(wikitext):
22 template_level = 0
23 new_wikitext = ""
24 while True:
25 assert template_level >= 0, ValueError("Unbalanced template in wikitext:\n" + wikitext)
26 pre_open, open_tag, post_open = wikitext.partition("{{")
27 pre_close, close_tag, post_close = wikitext.partition("}}")
28 if open_tag and (not close_tag or len(pre_open) < len(pre_close)):
29 # Template starts here ({{)
30 wikitext = post_open
31 if template_level == 0:
32 new_wikitext += pre_open
33 template_level += 1
34 elif close_tag:
35 # Template ends here (}})
36 wikitext = post_close
37 template_level -= 1
38 else:
39 # No more templates
40 assert template_level == 0, ValueError("Unbalanced template in wikitext:\n" + wikitext)
41 # The assertion below must be true on earth
42 assert open_tag == close_tag == "", RuntimeError("Cosmic radiation detected")
43 new_wikitext += wikitext
44 break
45
46 return new_wikitext
47
48
49 def process(wikitext):
50 wikitext = trim_templates(wikitext)
51 words = collections.OrderedDict()
52
53 def add_word(word):
54 for garbage in ("[", "]", "…", ":", ":", ")", ")", '"', "“", "”", "-{", "}-", "简称", "簡稱"):
55 word = word.replace(garbage, "")
56 words[word.strip()] = None
57
58 def add_words(word):
59 for word_separator in ("、", "/", "|", ",", "。", "?", "?", "(", "("):
60 if word_separator in word:
61 for w in word.split(word_separator):
62 # recursively resolve
63 add_words(w.strip())
64 break
65 else:
66 add_word(word)
67
68 def iter_bolds(line):
69 line_bak = line
70 while "'''" in line:
71 _, sep1, line = line.partition("'''")
72 bold, sep2, line = line.partition("'''")
73 assert sep1 and sep2, ValueError("Unclosed ''' in line: " + line_bak)
74 yield bold
75
76 for line in wikitext.split("\n"):
77 if not line.startswith("*"):
78 continue
79 # Lists
80 line = line.strip("*").strip()
81 pre_colon, sep, post_colon = line.partition("''':")
82 if not sep:
83 pre_colon, sep, post_colon = line.partition("''':")
84 for bold in iter_bolds(pre_colon + sep):
85 # Add bold words before colon
86 add_words(bold)
87 for bold in iter_bolds(post_colon):
88 # Add bold words after colon (or line w/o colon), skipping the origin of abbreviation (length probably <= 2)
89 if len(bold) > 2:
90 add_words(bold)
91
92 return words
93
94
95 def print_words(words):
96 for word in words:
97 print(word)
98
99
100 if __name__ == "__main__":
101 if len(sys.argv) == 1:
102 wikitext = fetch()
103 words = process(wikitext)
104 print_words(words)
105
106 elif sys.argv[1] == "--fetch":
107 print(fetch())
108
109 elif sys.argv[1] == "--process":
110 wikitext = open(sys.argv[2]).read()
111 print_words(process(wikitext))
112
113 else:
114 raise NotImplementedError