| 1 |
/* |
| 2 |
* SPDX-FileCopyrightText: 2012-2012 Yichao Yu <[email protected]> |
| 3 |
* SPDX-FileCopyrightText: 2017-2017 CSSlayer <[email protected]> |
| 4 |
* |
| 5 |
* SPDX-License-Identifier: LGPL-2.1-or-later |
| 6 |
* |
| 7 |
*/ |
| 8 |
#include "stroke.h" |
| 9 |
|
| 10 |
#include <cstddef> |
| 11 |
#include <cstdint> |
| 12 |
#include <fcitx-utils/fdstreambuf.h> |
| 13 |
#include <fcitx-utils/macros.h> |
| 14 |
#include <fcitx-utils/standardpaths.h> |
| 15 |
#include <fcitx-utils/stringutils.h> |
| 16 |
#include <fcitx-utils/utf8.h> |
| 17 |
#include <fcntl.h> |
| 18 |
#include <functional> |
| 19 |
#include <future> |
| 20 |
#include <istream> |
| 21 |
#include <libime/core/datrie.h> |
| 22 |
#include <optional> |
| 23 |
#include <queue> |
| 24 |
#include <stdexcept> |
| 25 |
#include <string> |
| 26 |
#include <string_view> |
| 27 |
#include <tuple> |
| 28 |
#include <unordered_set> |
| 29 |
#include <utility> |
| 30 |
#include <vector> |
| 31 |
|
| 32 |
namespace fcitx { |
| 33 |
|
| 34 |
Stroke::Stroke() {} |
| 35 |
|
| 36 |
void Stroke::loadAsync() { |
| 37 |
if (loadFuture_.valid()) { |
| 38 |
return; |
| 39 |
} |
| 40 |
|
| 41 |
loadFuture_ = std::async(std::launch::async, []() { |
| 42 |
std::tuple<libime::DATrie<int32_t>, libime::DATrie<int32_t>> result; |
| 43 |
auto &dict = std::get<0>(result); |
| 44 |
auto &reverseDict = std::get<1>(result); |
| 45 |
|
| 46 |
auto file = StandardPaths::global().open(StandardPathsType::PkgData, |
| 47 |
"pinyinhelper/py_stroke.mb"); |
| 48 |
if (!file.isValid()) { |
| 49 |
throw std::runtime_error("Failed to open file"); |
| 50 |
} |
| 51 |
|
| 52 |
IFDStreamBuf buffer(file.fd()); |
| 53 |
std::istream in(&buffer); |
| 54 |
std::string buf; |
| 55 |
while (!in.eof()) { |
| 56 |
if (!std::getline(in, buf)) { |
| 57 |
break; |
| 58 |
} |
| 59 |
// Validate everything first, so it's easier to process. |
| 60 |
if (!utf8::validate(buf)) { |
| 61 |
continue; |
| 62 |
} |
| 63 |
|
| 64 |
auto line = stringutils::trimView(buf); |
| 65 |
if (line.empty() || line[0] == '#') { |
| 66 |
continue; |
| 67 |
} |
| 68 |
auto pos = line.find_first_of(FCITX_WHITESPACE); |
| 69 |
if (pos == std::string::npos) { |
| 70 |
continue; |
| 71 |
} |
| 72 |
std::string_view key = line.substr(0, pos); |
| 73 |
std::string_view value = |
| 74 |
stringutils::trimView(line.substr(pos + 1)); |
| 75 |
if (utf8::length(value) != 1 || |
| 76 |
key.find_first_not_of("12345") != std::string::npos) { |
| 77 |
continue; |
| 78 |
} |
| 79 |
std::string token = stringutils::concat(key, "|", value); |
| 80 |
std::string reverseToken = stringutils::concat(value, "|", key); |
| 81 |
dict.set(token, 1); |
| 82 |
reverseDict.set(reverseToken, 1); |
| 83 |
} |
| 84 |
|
| 85 |
dict.shrink_tail(); |
| 86 |
reverseDict.shrink_tail(); |
| 87 |
|
| 88 |
return result; |
| 89 |
}); |
| 90 |
} |
| 91 |
|
| 92 |
bool Stroke::load() { |
| 93 |
if (loaded_) { |
| 94 |
return loadResult_; |
| 95 |
} |
| 96 |
if (!loadFuture_.valid()) { |
| 97 |
loadAsync(); |
| 98 |
} |
| 99 |
try { |
| 100 |
std::tie(dict_, revserseDict_) = loadFuture_.get(); |
| 101 |
loadResult_ = true; |
| 102 |
} catch (...) { |
| 103 |
loadResult_ = false; |
| 104 |
} |
| 105 |
|
| 106 |
loaded_ = true; |
| 107 |
return true; |
| 108 |
} |
| 109 |
|
| 110 |
#define DELETION_WEIGHT 5 |
| 111 |
#define INSERTION_WEIGHT 5 |
| 112 |
#define SUBSTITUTION_WEIGHT 5 |
| 113 |
#define TRANSPOSITION_WEIGHT 5 |
| 114 |
|
| 115 |
std::vector<std::pair<std::string, std::string>> |
| 116 |
Stroke::lookup(std::string_view input, int limit) { |
| 117 |
std::vector<std::pair<std::string, std::string>> result; |
| 118 |
std::unordered_set<std::string> resultSet; |
| 119 |
using position_type = decltype(dict_)::position_type; |
| 120 |
struct LookupItem { |
| 121 |
position_type pos; |
| 122 |
std::string_view remain; |
| 123 |
int weight; |
| 124 |
int length; |
| 125 |
|
| 126 |
bool operator>(const LookupItem &other) const { |
| 127 |
return weight > other.weight; |
| 128 |
} |
| 129 |
}; |
| 130 |
std::priority_queue<LookupItem, std::vector<LookupItem>, |
| 131 |
std::greater<LookupItem>> |
| 132 |
q; |
| 133 |
|
| 134 |
// First lets check if the stroke is already a prefix of single word. |
| 135 |
std::optional<position_type> onlyMatch; |
| 136 |
size_t onlyMatchLength = 0; |
| 137 |
auto addResult = [&result, &resultSet](std::string stroke, std::string hz) { |
| 138 |
if (resultSet.insert(hz).second) { |
| 139 |
result.emplace_back(std::move(stroke), std::move(hz)); |
| 140 |
} |
| 141 |
}; |
| 142 |
|
| 143 |
if (dict_.foreach(input, [&onlyMatch, &onlyMatchLength](int32_t, size_t len, |
| 144 |
uint64_t pos) { |
| 145 |
if (onlyMatch) { |
| 146 |
return false; |
| 147 |
} |
| 148 |
onlyMatch = pos; |
| 149 |
onlyMatchLength = len; |
| 150 |
return true; |
| 151 |
})) { |
| 152 |
if (onlyMatch) { |
| 153 |
std::string buf; |
| 154 |
dict_.suffix(buf, input.size() + onlyMatchLength, *onlyMatch); |
| 155 |
if (auto idx = buf.find_last_of('|'); idx != std::string::npos) { |
| 156 |
addResult(buf.substr(idx + 1), buf.substr(0, idx)); |
| 157 |
} |
| 158 |
} |
| 159 |
} |
| 160 |
if (result.size() >= static_cast<size_t>(limit)) { |
| 161 |
return result; |
| 162 |
} |
| 163 |
|
| 164 |
auto pushQueue = [&q](LookupItem &&item) { |
| 165 |
if (item.weight >= 10) { |
| 166 |
return; |
| 167 |
} |
| 168 |
q.push(item); |
| 169 |
}; |
| 170 |
|
| 171 |
pushQueue(LookupItem{.pos = 0, .remain = input, .weight = 0, .length = 0}); |
| 172 |
|
| 173 |
while (!q.empty()) { |
| 174 |
auto current = q.top(); |
| 175 |
q.pop(); |
| 176 |
if (current.remain.empty()) { |
| 177 |
if (!dict_.foreach( |
| 178 |
"|", |
| 179 |
[this, &result, ¤t, limit, |
| 180 |
&addResult](int32_t, size_t len, uint64_t pos) { |
| 181 |
std::string buf; |
| 182 |
dict_.suffix(buf, current.length + 1 + len, pos); |
| 183 |
addResult(buf.substr(current.length + 1), |
| 184 |
buf.substr(0, current.length)); |
| 185 |
return limit <= 0 || |
| 186 |
result.size() < static_cast<size_t>(limit); |
| 187 |
}, |
| 188 |
current.pos)) { |
| 189 |
break; |
| 190 |
} |
| 191 |
} |
| 192 |
|
| 193 |
// Deletion |
| 194 |
if (!current.remain.empty()) { |
| 195 |
pushQueue(LookupItem{.pos = current.pos, |
| 196 |
.remain = current.remain.substr(1), |
| 197 |
.weight = current.weight + DELETION_WEIGHT, |
| 198 |
.length = current.length}); |
| 199 |
} |
| 200 |
|
| 201 |
for (char i = '1'; i <= '5'; i++) { |
| 202 |
auto pos = current.pos; |
| 203 |
auto v = dict_.traverse(&i, 1, pos); |
| 204 |
if (libime::DATrie<int32_t>::isNoPath(v)) { |
| 205 |
continue; |
| 206 |
} |
| 207 |
if (!current.remain.empty() && current.remain[0] == i) { |
| 208 |
pushQueue(LookupItem{.pos = pos, |
| 209 |
.remain = current.remain.substr(1), |
| 210 |
.weight = current.weight, |
| 211 |
.length = current.length + 1}); |
| 212 |
} else { |
| 213 |
pushQueue( |
| 214 |
LookupItem{.pos = pos, |
| 215 |
.remain = current.remain, |
| 216 |
.weight = current.weight + INSERTION_WEIGHT, |
| 217 |
.length = current.length + 1}); |
| 218 |
if (!current.remain.empty()) { |
| 219 |
pushQueue(LookupItem{.pos = pos, |
| 220 |
.remain = current.remain.substr(1), |
| 221 |
.weight = current.weight + |
| 222 |
SUBSTITUTION_WEIGHT, |
| 223 |
.length = current.length + 1}); |
| 224 |
} |
| 225 |
} |
| 226 |
|
| 227 |
if (current.remain.size() >= 2 && current.remain[1] == i) { |
| 228 |
auto nextPos = pos; |
| 229 |
auto nextV = dict_.traverse(current.remain.data(), 1, nextPos); |
| 230 |
if (!libime::DATrie<int32_t>::isNoPath(nextV)) { |
| 231 |
pushQueue(LookupItem{nextPos, current.remain.substr(2), |
| 232 |
current.weight + TRANSPOSITION_WEIGHT, |
| 233 |
current.length + 2}); |
| 234 |
} |
| 235 |
} |
| 236 |
} |
| 237 |
} |
| 238 |
|
| 239 |
return result; |
| 240 |
} |
| 241 |
|
| 242 |
std::string Stroke::reverseLookup(const std::string &hanzi) const { |
| 243 |
using position_type = decltype(dict_)::position_type; |
| 244 |
libime::DATrie<int32_t>::position_type pos = 0; |
| 245 |
auto result = revserseDict_.traverse(hanzi, pos); |
| 246 |
if (libime::DATrie<int32_t>::isNoPath(result)) { |
| 247 |
return {}; |
| 248 |
} |
| 249 |
result = revserseDict_.traverse("|", pos); |
| 250 |
if (libime::DATrie<int32_t>::isNoPath(result)) { |
| 251 |
return {}; |
| 252 |
} |
| 253 |
std::optional<position_type> onlyMatch; |
| 254 |
size_t onlyMatchLength = 0; |
| 255 |
if (revserseDict_.foreach( |
| 256 |
[&onlyMatch, &onlyMatchLength](int32_t, size_t len, uint64_t pos) { |
| 257 |
if (onlyMatch) { |
| 258 |
return false; |
| 259 |
} |
| 260 |
onlyMatch = pos; |
| 261 |
onlyMatchLength = len; |
| 262 |
return true; |
| 263 |
}, |
| 264 |
pos)) { |
| 265 |
if (onlyMatch) { |
| 266 |
std::string buf; |
| 267 |
revserseDict_.suffix(buf, onlyMatchLength, *onlyMatch); |
| 268 |
return buf; |
| 269 |
} |
| 270 |
} |
| 271 |
return {}; |
| 272 |
} |
| 273 |
|
| 274 |
std::string Stroke::prettyString(const std::string &input) const { |
| 275 |
std::string result; |
| 276 |
static const std::string_view stroke_table[] = {"一", "丨", "丿", |
| 277 |
"㇏", "𠃍", ""}; |
| 278 |
for (auto c : input) { |
| 279 |
if (c < '1' || c > '5') { |
| 280 |
return {}; |
| 281 |
} |
| 282 |
auto v = stroke_table[c - '1']; |
| 283 |
result.append(v.begin(), v.end()); |
| 284 |
} |
| 285 |
return result; |
| 286 |
} |
| 287 |
} // namespace fcitx |