Debian Codegraph @

src/util/dtextencoding.cpp

raw
1 // SPDX-FileCopyrightText: 2022-2023 UnionTech Software Technology Co., Ltd.
2 //
3 // SPDX-License-Identifier: LGPL-3.0-or-later
4
5 #include "dtextencoding.h"
6
7 #include <QtMath>
8 #include <QFile>
9 #include <QLibrary>
10 #if QT_VERSION >= QT_VERSION_CHECK(6, 0, 0)
11 #include <QStringConverter>
12 #else
13 #include <QTextCodec>
14 #endif
15
16 #include <climits>
17 #include <unicode/ucsdet.h>
18 #include <uchardet/uchardet.h>
19 #include <iconv.h>
20
21 DCORE_BEGIN_NAMESPACE
22
23 class LibICU
24 {
25 public:
26 LibICU();
27 ~LibICU();
28
29 bool isValid();
30 bool detectEncoding(const QByteArray &content, QByteArrayList &charset);
31
32 UCharsetDetector *(*icu_ucsdet_open)(UErrorCode *status);
33 void (*icu_ucsdet_close)(UCharsetDetector *ucsd);
34 void (*icu_ucsdet_setText)(UCharsetDetector *ucsd, const char *textIn, int32_t len, UErrorCode *status);
35 const UCharsetMatch **(*icu_ucsdet_detectAll)(UCharsetDetector *ucsd, int32_t *matchesFound, UErrorCode *status);
36 const char *(*icu_ucsdet_getName)(const UCharsetMatch *ucsm, UErrorCode *status);
37 int32_t (*icu_ucsdet_getConfidence)(const UCharsetMatch *ucsm, UErrorCode *status);
38
39 private:
40 QLibrary *icuuc = nullptr;
41
42 Q_DISABLE_COPY(LibICU)
43 };
44
45 class Libuchardet
46 {
47 public:
48 Libuchardet();
49 ~Libuchardet();
50
51 bool isValid();
52 QByteArray detectEncoding(const QByteArray &content);
53
54 uchardet_t (*uchardet_new)(void);
55 void (*uchardet_delete)(uchardet_t ud);
56 int (*uchardet_handle_data)(uchardet_t ud, const char *data, size_t len);
57 void (*uchardet_data_end)(uchardet_t ud);
58 void (*uchardet_reset)(uchardet_t ud);
59 const char *(*uchardet_get_charset)(uchardet_t ud);
60
61 private:
62 QLibrary *uchardet = nullptr;
63
64 Q_DISABLE_COPY(Libuchardet)
65 };
66
67 Q_GLOBAL_STATIC(LibICU, LibICUInstance);
68 Q_GLOBAL_STATIC(Libuchardet, LibuchardetInstance);
69
70 LibICU::LibICU()
71 {
72 // Load libicuuc.so
73 icuuc = new QLibrary("libicuuc");
74 if (!icuuc->load()) {
75 delete icuuc;
76 icuuc = nullptr;
77 return;
78 }
79
80 auto initFunctionError = [this]() {
81 icuuc->unload();
82 delete icuuc;
83 icuuc = nullptr;
84 };
85
86 #define INIT_ICUUC(Name) \
87 icu_##Name = reinterpret_cast<decltype(icu_##Name)>(icuuc->resolve(#Name)); \
88 if (!icu_##Name) { \
89 initFunctionError(); \
90 return; \
91 }
92
93 // Note: use prefix 'icu' to avoid "disabled expansion of recursive macro" warning.
94 INIT_ICUUC(ucsdet_open);
95 INIT_ICUUC(ucsdet_close);
96 INIT_ICUUC(ucsdet_setText);
97 INIT_ICUUC(ucsdet_detectAll);
98 INIT_ICUUC(ucsdet_getName);
99 INIT_ICUUC(ucsdet_getConfidence);
100 }
101
102 LibICU::~LibICU()
103 {
104 if (icuuc) {
105 delete icuuc;
106 }
107 }
108
109 bool LibICU::isValid()
110 {
111 return (icuuc);
112 }
113
114 bool LibICU::detectEncoding(const QByteArray &content, QByteArrayList &charset)
115 {
116 UErrorCode status = U_ZERO_ERROR;
117 UCharsetDetector *detector = icu_ucsdet_open(&status);
118 if (U_FAILURE(status)) {
119 return false;
120 }
121
122 icu_ucsdet_setText(detector, content.data(), content.size(), &status);
123 if (U_FAILURE(status)) {
124 icu_ucsdet_close(detector);
125 return false;
126 }
127
128 int32_t matchCount = 0;
129 const UCharsetMatch **charsetMatch = icu_ucsdet_detectAll(detector, &matchCount, &status);
130 if (U_FAILURE(status)) {
131 icu_ucsdet_close(detector);
132 return false;
133 }
134
135 int recordCount = qMin(3, matchCount);
136 for (int i = 0; i < recordCount; i++) {
137 const char *encoding = icu_ucsdet_getName(charsetMatch[i], &status);
138 if (U_FAILURE(status)) {
139 icu_ucsdet_close(detector);
140 return false;
141 }
142 charset << QByteArray(encoding);
143 }
144
145 icu_ucsdet_close(detector);
146 return true;
147 }
148
149 Libuchardet::Libuchardet()
150 {
151 uchardet = new QLibrary("libuchardet", "0");
152 if (!uchardet->load()) {
153 delete uchardet;
154 uchardet = nullptr;
155 return;
156 }
157
158 auto initFunctionError = [this]() {
159 uchardet->unload();
160 delete uchardet;
161 uchardet = nullptr;
162 };
163
164 #define INIT_UCHARDET(Name) \
165 Name = reinterpret_cast<decltype(Name)>(uchardet->resolve(#Name)); \
166 if (!Name) { \
167 initFunctionError(); \
168 return; \
169 }
170
171 INIT_UCHARDET(uchardet_new);
172 INIT_UCHARDET(uchardet_delete);
173 INIT_UCHARDET(uchardet_handle_data);
174 INIT_UCHARDET(uchardet_data_end);
175 INIT_UCHARDET(uchardet_reset);
176 INIT_UCHARDET(uchardet_get_charset);
177 }
178
179 Libuchardet::~Libuchardet()
180 {
181 if (uchardet) {
182 delete uchardet;
183 }
184 }
185
186 bool Libuchardet::isValid()
187 {
188 return uchardet;
189 }
190
191 QByteArray Libuchardet::detectEncoding(const QByteArray &content)
192 {
193 QByteArray charset;
194
195 uchardet_t handle = uchardet_new();
196 if (0 == uchardet_handle_data(handle, content.data(), static_cast<size_t>(content.size()))) {
197 uchardet_data_end(handle);
198 charset = QByteArray(uchardet_get_charset(handle));
199 }
200 uchardet_delete(handle);
201
202 return charset;
203 }
204
205 QByteArray selectCharset(const QByteArray &charset, const QByteArrayList &icuCharsetList)
206 {
207 if (icuCharsetList.isEmpty()) {
208 return charset;
209 }
210
211 static QByteArray encodingGB18030("GB18030");
212 if (charset.isEmpty()) {
213 return icuCharsetList.contains(encodingGB18030) ? encodingGB18030 : icuCharsetList[0];
214 } else {
215 if (charset.contains(icuCharsetList[0])) {
216 return charset;
217 } else {
218 return icuCharsetList[0].contains(charset) ? icuCharsetList[0] : charset;
219 }
220 }
221 }
222
223 QByteArray DTextEncoding::detectTextEncoding(const QByteArray &content)
224 {
225 if (content.isEmpty()) {
226 return QByteArray("UTF-8");
227 }
228
229 QByteArray charset;
230 if (LibuchardetInstance()->isValid()) {
231 charset = LibuchardetInstance()->detectEncoding(content);
232 }
233
234 if (LibICUInstance()->isValid()) {
235 QByteArrayList icuCharsetList;
236 if (LibICUInstance()->detectEncoding(content, icuCharsetList)) {
237 if (charset.isEmpty() && !icuCharsetList.isEmpty()) {
238 charset = icuCharsetList.first();
239 } else {
240 // Improve GB18030 encoding recognition rate.
241 charset = selectCharset(charset, icuCharsetList);
242 }
243 }
244 }
245
246 if (charset.isEmpty()) {
247 #if QT_VERSION >= QT_VERSION_CHECK(6, 0, 0)
248 std::optional<QStringConverter::Encoding> encoding = QStringConverter::encodingForData(content);
249 if (encoding) {
250 return QStringConverter::nameForEncoding(encoding.value());
251 }
252 #else
253 QTextCodec *codec = QTextCodec::codecForUtfText(content);
254 if (codec) {
255 return codec->name();
256 }
257 #endif
258 }
259
260 // Use default encoding.
261 if (charset.isEmpty() || charset.contains("ASCII")) {
262 charset = QByteArray("UTF-8");
263 }
264
265 return charset;
266 }
267
268 QByteArray DTextEncoding::detectFileEncoding(const QString &fileName, bool *isOk)
269 {
270 QFile file(fileName);
271 if (!file.open(QFile::ReadOnly | QFile::Text)) {
272 if (isOk) {
273 *isOk = false;
274 }
275 return QByteArray();
276 }
277
278 // At most 64Kb data.
279 QByteArray content = file.read(qMin<int>(static_cast<int>(file.size()), USHRT_MAX));
280 file.close();
281
282 if (isOk) {
283 *isOk = true;
284 }
285 return detectTextEncoding(content);
286 }
287
288 bool DTextEncoding::convertTextEncoding(
289 QByteArray &content, QByteArray &outContent, const QByteArray &toEncoding, const QByteArray &fromEncoding, QString *errString)
290 {
291 return convertTextEncodingEx(content, outContent, toEncoding, fromEncoding, errString);
292 }
293
294 bool DTextEncoding::convertTextEncodingEx(QByteArray &content,
295 QByteArray &outContent,
296 const QByteArray &toEncoding,
297 const QByteArray &fromEncoding,
298 QString *errString,
299 int *convertedBytes)
300 {
301 if (content.isEmpty() || fromEncoding == toEncoding) {
302 return true;
303 }
304
305 if (toEncoding.isEmpty()) {
306 if (errString) {
307 *errString = QStringLiteral("The encode that convert to is empty.");
308 }
309 return false;
310 }
311
312 QByteArray contentEncoding = fromEncoding;
313 if (contentEncoding.isEmpty()) {
314 contentEncoding = detectTextEncoding(content);
315 }
316
317 // iconv set errno when failed.
318 iconv_t handle = iconv_open(toEncoding.data(), contentEncoding.data());
319 if (reinterpret_cast<iconv_t>(-1) != handle) {
320 size_t inBytesLeft = static_cast<size_t>(content.size());
321 char *inbuf = content.data();
322 size_t outBytesLeft = inBytesLeft * 4;
323 char *outbuf = new char[outBytesLeft];
324
325 char *bufferHeader = outbuf;
326 size_t maxBufferSize = outBytesLeft;
327
328 size_t ret = iconv(handle, &inbuf, &inBytesLeft, &outbuf, &outBytesLeft);
329 int convertError = 0;
330 if (static_cast<size_t>(-1) == ret) {
331 convertError = errno;
332 int converted = content.size() - static_cast<int>(inBytesLeft);
333 if (convertedBytes) {
334 *convertedBytes = converted;
335 }
336
337 if (errString) {
338 switch (convertError) {
339 case EILSEQ:
340 *errString = QString("An invalid multibyte sequence has been encountered in the input."
341 "Converted byte index: %1")
342 .arg(converted);
343 break;
344 case EINVAL:
345 *errString = QString("An incomplete multibyte sequence has been encountered in the input. "
346 "Converted byte index: %1")
347 .arg(converted);
348 break;
349 case E2BIG:
350 *errString = QString("There is not sufficient room at *outbuf. Converted byte index: %1").arg(converted);
351 break;
352 default:
353 break;
354 }
355 }
356 }
357 iconv_close(handle);
358
359 // Use iconv converted byte count.
360 size_t realConvertSize = maxBufferSize - outBytesLeft;
361 outContent = QByteArray(bufferHeader, static_cast<int>(realConvertSize));
362
363 delete[] bufferHeader;
364 // For errors, user decides to keep or remove converted text.
365 return 0 == convertError;
366
367 } else {
368 if (EINVAL == errno && errString) {
369 *errString = QStringLiteral("The conversion from fromcode to tocode is not supported by the implementation.");
370 }
371
372 return false;
373 }
374 }
375
376 bool DTextEncoding::convertFileEncoding(const QString &fileName,
377 const QByteArray &toEncoding,
378 const QByteArray &fromEncoding,
379 QString *errString)
380 {
381 if (fromEncoding == toEncoding) {
382 return true;
383 }
384
385 QFile file(fileName);
386 if (!file.open(QFile::ReadWrite | QFile::Text)) {
387 if (errString) {
388 *errString = file.errorString();
389 file.error();
390 }
391 return false;
392 }
393
394 QByteArray content = file.readAll();
395 QByteArray outContent;
396 if (!convertTextEncoding(content, outContent, toEncoding, fromEncoding, errString)) {
397 file.close();
398 return false;
399 }
400
401 file.seek(0);
402 file.write(outContent);
403 file.resize(outContent.size());
404 file.close();
405
406 if (QFile::NoError != file.error()) {
407 if (errString) {
408 *errString = file.errorString();
409 }
410 return false;
411 }
412 return true;
413 }
414
415 bool DTextEncoding::convertFileEncodingTo(const QString &fromFile,
416 const QString &toFile,
417 const QByteArray &toEncoding,
418 const QByteArray &fromEncoding,
419 QString *errString)
420 {
421 if (fromEncoding == toEncoding) {
422 return true;
423 }
424
425 if (fromFile == toFile) {
426 return convertFileEncoding(fromFile, toEncoding, fromEncoding, errString);
427 }
428
429 // Check from file and to file before convert.
430 QFile readFile(fromFile);
431 if (!readFile.open(QFile::ReadOnly | QFile::Text)) {
432 if (errString) {
433 *errString = QString("Open convert from file failed, %1").arg(readFile.errorString());
434 }
435 return false;
436 }
437
438 QFile writeFile(toFile);
439 if (!writeFile.open(QFile::WriteOnly | QFile::Text)) {
440 readFile.close();
441 if (errString) {
442 *errString = QString("Open convert to file failed, %1").arg(writeFile.errorString());
443 }
444 return false;
445 }
446
447 QByteArray content = readFile.readAll();
448 readFile.close();
449 QByteArray outContent;
450
451 if (!convertTextEncoding(content, outContent, toEncoding, fromEncoding, errString)) {
452 writeFile.close();
453 writeFile.remove();
454 return false;
455 }
456
457 writeFile.write(outContent);
458 writeFile.close();
459
460 if (QFile::NoError != writeFile.error()) {
461 if (errString) {
462 *errString = writeFile.errorString();
463 }
464 return false;
465 }
466 return true;
467 }
468
469 DCORE_END_NAMESPACE