Debian Codegraph @

modules/pinyinhelper/stroke.cpp

raw
1 /*
2 * SPDX-FileCopyrightText: 2012-2012 Yichao Yu <[email protected]>
3 * SPDX-FileCopyrightText: 2017-2017 CSSlayer <[email protected]>
4 *
5 * SPDX-License-Identifier: LGPL-2.1-or-later
6 *
7 */
8 #include "stroke.h"
9
10 #include <cstddef>
11 #include <cstdint>
12 #include <fcitx-utils/fdstreambuf.h>
13 #include <fcitx-utils/macros.h>
14 #include <fcitx-utils/standardpaths.h>
15 #include <fcitx-utils/stringutils.h>
16 #include <fcitx-utils/utf8.h>
17 #include <fcntl.h>
18 #include <functional>
19 #include <future>
20 #include <istream>
21 #include <libime/core/datrie.h>
22 #include <optional>
23 #include <queue>
24 #include <stdexcept>
25 #include <string>
26 #include <string_view>
27 #include <tuple>
28 #include <unordered_set>
29 #include <utility>
30 #include <vector>
31
32 namespace fcitx {
33
34 Stroke::Stroke() {}
35
36 void Stroke::loadAsync() {
37 if (loadFuture_.valid()) {
38 return;
39 }
40
41 loadFuture_ = std::async(std::launch::async, []() {
42 std::tuple<libime::DATrie<int32_t>, libime::DATrie<int32_t>> result;
43 auto &dict = std::get<0>(result);
44 auto &reverseDict = std::get<1>(result);
45
46 auto file = StandardPaths::global().open(StandardPathsType::PkgData,
47 "pinyinhelper/py_stroke.mb");
48 if (!file.isValid()) {
49 throw std::runtime_error("Failed to open file");
50 }
51
52 IFDStreamBuf buffer(file.fd());
53 std::istream in(&buffer);
54 std::string buf;
55 while (!in.eof()) {
56 if (!std::getline(in, buf)) {
57 break;
58 }
59 // Validate everything first, so it's easier to process.
60 if (!utf8::validate(buf)) {
61 continue;
62 }
63
64 auto line = stringutils::trimView(buf);
65 if (line.empty() || line[0] == '#') {
66 continue;
67 }
68 auto pos = line.find_first_of(FCITX_WHITESPACE);
69 if (pos == std::string::npos) {
70 continue;
71 }
72 std::string_view key = line.substr(0, pos);
73 std::string_view value =
74 stringutils::trimView(line.substr(pos + 1));
75 if (utf8::length(value) != 1 ||
76 key.find_first_not_of("12345") != std::string::npos) {
77 continue;
78 }
79 std::string token = stringutils::concat(key, "|", value);
80 std::string reverseToken = stringutils::concat(value, "|", key);
81 dict.set(token, 1);
82 reverseDict.set(reverseToken, 1);
83 }
84
85 dict.shrink_tail();
86 reverseDict.shrink_tail();
87
88 return result;
89 });
90 }
91
92 bool Stroke::load() {
93 if (loaded_) {
94 return loadResult_;
95 }
96 if (!loadFuture_.valid()) {
97 loadAsync();
98 }
99 try {
100 std::tie(dict_, revserseDict_) = loadFuture_.get();
101 loadResult_ = true;
102 } catch (...) {
103 loadResult_ = false;
104 }
105
106 loaded_ = true;
107 return true;
108 }
109
110 #define DELETION_WEIGHT 5
111 #define INSERTION_WEIGHT 5
112 #define SUBSTITUTION_WEIGHT 5
113 #define TRANSPOSITION_WEIGHT 5
114
115 std::vector<std::pair<std::string, std::string>>
116 Stroke::lookup(std::string_view input, int limit) {
117 std::vector<std::pair<std::string, std::string>> result;
118 std::unordered_set<std::string> resultSet;
119 using position_type = decltype(dict_)::position_type;
120 struct LookupItem {
121 position_type pos;
122 std::string_view remain;
123 int weight;
124 int length;
125
126 bool operator>(const LookupItem &other) const {
127 return weight > other.weight;
128 }
129 };
130 std::priority_queue<LookupItem, std::vector<LookupItem>,
131 std::greater<LookupItem>>
132 q;
133
134 // First lets check if the stroke is already a prefix of single word.
135 std::optional<position_type> onlyMatch;
136 size_t onlyMatchLength = 0;
137 auto addResult = [&result, &resultSet](std::string stroke, std::string hz) {
138 if (resultSet.insert(hz).second) {
139 result.emplace_back(std::move(stroke), std::move(hz));
140 }
141 };
142
143 if (dict_.foreach(input, [&onlyMatch, &onlyMatchLength](int32_t, size_t len,
144 uint64_t pos) {
145 if (onlyMatch) {
146 return false;
147 }
148 onlyMatch = pos;
149 onlyMatchLength = len;
150 return true;
151 })) {
152 if (onlyMatch) {
153 std::string buf;
154 dict_.suffix(buf, input.size() + onlyMatchLength, *onlyMatch);
155 if (auto idx = buf.find_last_of('|'); idx != std::string::npos) {
156 addResult(buf.substr(idx + 1), buf.substr(0, idx));
157 }
158 }
159 }
160 if (result.size() >= static_cast<size_t>(limit)) {
161 return result;
162 }
163
164 auto pushQueue = [&q](LookupItem &&item) {
165 if (item.weight >= 10) {
166 return;
167 }
168 q.push(item);
169 };
170
171 pushQueue(LookupItem{.pos = 0, .remain = input, .weight = 0, .length = 0});
172
173 while (!q.empty()) {
174 auto current = q.top();
175 q.pop();
176 if (current.remain.empty()) {
177 if (!dict_.foreach(
178 "|",
179 [this, &result, &current, limit,
180 &addResult](int32_t, size_t len, uint64_t pos) {
181 std::string buf;
182 dict_.suffix(buf, current.length + 1 + len, pos);
183 addResult(buf.substr(current.length + 1),
184 buf.substr(0, current.length));
185 return limit <= 0 ||
186 result.size() < static_cast<size_t>(limit);
187 },
188 current.pos)) {
189 break;
190 }
191 }
192
193 // Deletion
194 if (!current.remain.empty()) {
195 pushQueue(LookupItem{.pos = current.pos,
196 .remain = current.remain.substr(1),
197 .weight = current.weight + DELETION_WEIGHT,
198 .length = current.length});
199 }
200
201 for (char i = '1'; i <= '5'; i++) {
202 auto pos = current.pos;
203 auto v = dict_.traverse(&i, 1, pos);
204 if (libime::DATrie<int32_t>::isNoPath(v)) {
205 continue;
206 }
207 if (!current.remain.empty() && current.remain[0] == i) {
208 pushQueue(LookupItem{.pos = pos,
209 .remain = current.remain.substr(1),
210 .weight = current.weight,
211 .length = current.length + 1});
212 } else {
213 pushQueue(
214 LookupItem{.pos = pos,
215 .remain = current.remain,
216 .weight = current.weight + INSERTION_WEIGHT,
217 .length = current.length + 1});
218 if (!current.remain.empty()) {
219 pushQueue(LookupItem{.pos = pos,
220 .remain = current.remain.substr(1),
221 .weight = current.weight +
222 SUBSTITUTION_WEIGHT,
223 .length = current.length + 1});
224 }
225 }
226
227 if (current.remain.size() >= 2 && current.remain[1] == i) {
228 auto nextPos = pos;
229 auto nextV = dict_.traverse(current.remain.data(), 1, nextPos);
230 if (!libime::DATrie<int32_t>::isNoPath(nextV)) {
231 pushQueue(LookupItem{nextPos, current.remain.substr(2),
232 current.weight + TRANSPOSITION_WEIGHT,
233 current.length + 2});
234 }
235 }
236 }
237 }
238
239 return result;
240 }
241
242 std::string Stroke::reverseLookup(const std::string &hanzi) const {
243 using position_type = decltype(dict_)::position_type;
244 libime::DATrie<int32_t>::position_type pos = 0;
245 auto result = revserseDict_.traverse(hanzi, pos);
246 if (libime::DATrie<int32_t>::isNoPath(result)) {
247 return {};
248 }
249 result = revserseDict_.traverse("|", pos);
250 if (libime::DATrie<int32_t>::isNoPath(result)) {
251 return {};
252 }
253 std::optional<position_type> onlyMatch;
254 size_t onlyMatchLength = 0;
255 if (revserseDict_.foreach(
256 [&onlyMatch, &onlyMatchLength](int32_t, size_t len, uint64_t pos) {
257 if (onlyMatch) {
258 return false;
259 }
260 onlyMatch = pos;
261 onlyMatchLength = len;
262 return true;
263 },
264 pos)) {
265 if (onlyMatch) {
266 std::string buf;
267 revserseDict_.suffix(buf, onlyMatchLength, *onlyMatch);
268 return buf;
269 }
270 }
271 return {};
272 }
273
274 std::string Stroke::prettyString(const std::string &input) const {
275 std::string result;
276 static const std::string_view stroke_table[] = {"一", "丨", "丿",
277 "㇏", "𠃍", ""};
278 for (auto c : input) {
279 if (c < '1' || c > '5') {
280 return {};
281 }
282 auto v = stroke_table[c - '1'];
283 result.append(v.begin(), v.end());
284 }
285 return result;
286 }
287 } // namespace fcitx