Debian Codegraph @

tests/ut_dtextencoding.cpp

raw
1 // SPDX-FileCopyrightText: 2022-2023 UnionTech Software Technology Co., Ltd.
2 //
3 // SPDX-License-Identifier: LGPL-3.0-or-later
4
5 #include "dtextencoding.h"
6
7 #include <gtest/gtest.h>
8
9 #include <QLibrary>
10 #include <QFile>
11 #include <QTemporaryFile>
12
13 #if QT_VERSION >= QT_VERSION_CHECK(6, 0, 0)
14 #include <QStringConverter>
15 #else
16 #include <QTextCodec>
17 #endif
18
19 DCORE_USE_NAMESPACE
20
21 class ut_DTextEncoding : public testing::Test
22 {
23 public:
24 static void SetUpTestCase();
25
26 bool rewriteTempFile(const QByteArray &data);
27 void removeTempFile();
28
29 static bool canLoadUchardet;
30 static bool canLoadICU;
31 static QString tmpFileName;
32
33 static QByteArray dataGB18030;
34 static QByteArray dataEUC_JP;
35 static QByteArray dataKOI8_R;
36
37 protected:
38 void TearDown() override;
39 };
40
41 bool ut_DTextEncoding::canLoadUchardet = false;
42 bool ut_DTextEncoding::canLoadICU = false;
43 QString ut_DTextEncoding::tmpFileName;
44 // Utf8 Chinese Text(Hex): 中文测试一二三四123456789abcdefgh
45 QByteArray ut_DTextEncoding::dataGB18030 = "\xd6\xd0\xce\xc4\xb2\xe2\xca\xd4\xd2\xbb\xb6\xfe\xc8\xfd\xcb\xc4\x31\x32\x33\x34\x35"
46 "\x36\x37\x38\x39\x61\x62\x63\x64\x65\x66\x67\x68";
47 // Utf8 Japanese Text(Hex): 日本語のテスト ワン ツー スリー フォー 123456789abcdefgh
48 QByteArray ut_DTextEncoding::dataEUC_JP =
49 "\xc6\xfc\xcb\xdc\xb8\xec\xa4\xce\xa5\xc6\xa5\xb9\xa5\xc8\x20\xa5\xef\xa5\xf3\x20\xa5\xc4\xa1\xbc\x20\xa5\xb9\xa5\xea\xa1\xbc"
50 "\x20\xa5\xd5\xa5\xa9\xa1\xbc\x20\x31\x32\x33\x34\x35\x36\x37\x38\x39\x61\x62\x63\x64\x65\x66\x67\x68";
51 // Utf8 Russian Text(Hex): Русский тест раз два три четыре 123456789abcdefgh
52 QByteArray ut_DTextEncoding::dataKOI8_R =
53 "\xf2\xd5\xd3\xd3\xcb\xc9\xca\x20\xd4\xc5\xd3\xd4\x20\xd2\xc1\xda\x20\xc4\xd7\xc1\x20\xd4\xd2\xc9\x20\xde\xc5\xd4\xd9\xd2\xc5"
54 "\x20\x31\x32\x33\x34\x35\x36\x37\x38\x39\x61\x62\x63\x64\x65\x66\x67\x68";
55
56 void ut_DTextEncoding::SetUpTestCase()
57 {
58 QLibrary uchardet("libuchardet", "0");
59 if (!uchardet.isLoaded()) {
60 canLoadUchardet = uchardet.load();
61 if (canLoadUchardet) {
62 uchardet.unload();
63 }
64 }
65
66 QLibrary icuuc("libicuuc");
67 if (icuuc.isLoaded()) {
68 canLoadICU = icuuc.load();
69 if (canLoadICU) {
70 icuuc.unload();
71 }
72 }
73 }
74
75 bool ut_DTextEncoding::rewriteTempFile(const QByteArray &data)
76 {
77 QTemporaryFile tmpFile;
78 if (!tmpFile.open()) {
79 return false;
80 }
81 tmpFileName = tmpFile.fileName();
82 tmpFile.setAutoRemove(false);
83
84 tmpFile.write(data);
85 tmpFile.close();
86 return true;
87 }
88
89 void ut_DTextEncoding::removeTempFile()
90 {
91 if (QFile::exists(tmpFileName)) {
92 QFile::remove(tmpFileName);
93 }
94 }
95
96 void ut_DTextEncoding::TearDown()
97 {
98 removeTempFile();
99 }
100
101 TEST_F(ut_DTextEncoding, testDetectTextEncode)
102 {
103 // Default encoding is utf-8.
104 ASSERT_EQ("UTF-8", DTextEncoding::detectTextEncoding(""));
105 ASSERT_EQ("UTF-8", DTextEncoding::detectTextEncoding("12345678ABCDEFG"));
106
107 ASSERT_EQ("GB18030", DTextEncoding::detectTextEncoding(dataGB18030));
108 ASSERT_EQ("EUC-JP", DTextEncoding::detectTextEncoding(dataEUC_JP));
109 ASSERT_EQ("KOI8-R", DTextEncoding::detectTextEncoding(dataKOI8_R));
110 }
111
112 TEST_F(ut_DTextEncoding, testDetectTextEncodeWithUchardet)
113 {
114 if (canLoadUchardet) {
115 QByteArray uchardetEncoding("EUC-TW");
116 #if QT_VERSION >= QT_VERSION_CHECK(6, 0, 0)
117 auto encode = QStringConverter::encodingForName(uchardetEncoding);
118 ASSERT_FALSE(encode);
119 #else
120 // QTextCodec not suppotted EUC-TW.
121 QTextCodec *codec = QTextCodec::codecForName(uchardetEncoding);
122 ASSERT_EQ(codec, nullptr);
123 #endif
124
125 // Utf8 text: 繁體中文測試一二三四
126 QByteArray dataZhTraditional("\u7e41\u9ad4\u4e2d\u6587\u6e2c\u8a66\u4e00\u4e8c\u4e09\u56db");
127 QByteArray dataEUC_TW;
128 ASSERT_TRUE(DTextEncoding::convertTextEncoding(dataZhTraditional, dataEUC_TW, uchardetEncoding));
129 ASSERT_EQ(uchardetEncoding, DTextEncoding::detectTextEncoding(dataEUC_TW));
130
131 QByteArray convertGB18030;
132 ASSERT_TRUE(DTextEncoding::convertTextEncoding(dataEUC_TW, convertGB18030, "GB18030"));
133 QByteArray convertUTF8;
134 ASSERT_TRUE(DTextEncoding::convertTextEncoding(convertGB18030, convertUTF8, "UTF-8"));
135 ASSERT_EQ(dataZhTraditional, convertUTF8);
136 }
137 }
138
139 TEST_F(ut_DTextEncoding, testDetectFileEncode)
140 {
141 bool isOk = false;
142 ASSERT_EQ("", DTextEncoding::detectFileEncoding(tmpFileName, &isOk));
143 ASSERT_FALSE(isOk);
144
145 ASSERT_TRUE(rewriteTempFile(""));
146 ASSERT_EQ("UTF-8", DTextEncoding::detectFileEncoding(tmpFileName, &isOk));
147 ASSERT_TRUE(isOk);
148
149 ASSERT_TRUE(rewriteTempFile(dataGB18030));
150 ASSERT_EQ("GB18030", DTextEncoding::detectFileEncoding(tmpFileName, &isOk));
151 ASSERT_TRUE(isOk);
152
153 ASSERT_TRUE(rewriteTempFile(dataEUC_JP));
154 ASSERT_EQ("EUC-JP", DTextEncoding::detectFileEncoding(tmpFileName, &isOk));
155 ASSERT_TRUE(isOk);
156
157 ASSERT_TRUE(rewriteTempFile(dataKOI8_R));
158 ASSERT_EQ("KOI8-R", DTextEncoding::detectFileEncoding(tmpFileName, &isOk));
159 ASSERT_TRUE(isOk);
160 }
161
162 TEST_F(ut_DTextEncoding, testConvertTextEncoding)
163 {
164 QByteArray dataUTF_8;
165 ASSERT_TRUE(DTextEncoding::convertTextEncoding(dataGB18030, dataUTF_8, "UTF-8"));
166 ASSERT_EQ("UTF-8", DTextEncoding::detectTextEncoding(dataUTF_8));
167
168 // QStringConverter not support GB18030.
169 #if QT_VERSION < QT_VERSION_CHECK(6, 0, 0)
170 QTextCodec *codec = QTextCodec::codecForName("GB18030");
171 ASSERT_EQ(codec->toUnicode(dataGB18030).toUtf8(), dataUTF_8);
172
173 QByteArray convertedGB18030;
174 ASSERT_TRUE(DTextEncoding::convertTextEncoding(dataUTF_8, convertedGB18030, "GB18030"));
175 ASSERT_EQ(dataGB18030, convertedGB18030);
176 #endif
177
178 // Convert with multi bytes encoding.
179 QByteArray dataUTF_16;
180 ASSERT_TRUE(DTextEncoding::convertTextEncoding(dataUTF_8, dataUTF_16, "UTF-16"));
181 ASSERT_EQ("UTF-16", DTextEncoding::detectTextEncoding(dataUTF_16));
182
183 #if QT_VERSION >= QT_VERSION_CHECK(6, 0, 0)
184 // Qt6 using utf-16 string by default, not utf-8.
185 auto fromUtf16 = QStringDecoder(QStringDecoder::Utf16);
186 QString strUtf16 = fromUtf16.decode(dataUTF_16);
187 ASSERT_EQ(strUtf16, QString::fromUtf8(dataUTF_8));
188 #else
189 codec = QTextCodec::codecForName("UTF-16");
190 ASSERT_EQ(codec->toUnicode(dataUTF_16).toUtf8(), dataUTF_8);
191 #endif
192
193 QByteArray convertedUTF8;
194 ASSERT_TRUE(DTextEncoding::convertTextEncoding(dataUTF_16, convertedUTF8, "UTF-8"));
195 ASSERT_EQ(dataUTF_8, convertedUTF8);
196
197 QByteArray dataUTF_32;
198 ASSERT_TRUE(DTextEncoding::convertTextEncoding(dataUTF_8, dataUTF_32, "UTF-32"));
199 ASSERT_EQ("UTF-32", DTextEncoding::detectTextEncoding(dataUTF_32));
200 #if QT_VERSION >= QT_VERSION_CHECK(6, 0, 0)
201 auto fromUtf32 = QStringDecoder(QStringDecoder::Utf32);
202 QString strUtf32 = fromUtf32.decode(dataUTF_32);
203 ASSERT_EQ(strUtf32, QString::fromUtf8(dataUTF_8));
204 #else
205 codec = QTextCodec::codecForName("UTF-32");
206 ASSERT_EQ(codec->toUnicode(dataUTF_32).toUtf8(), dataUTF_8);
207 #endif
208
209 ASSERT_TRUE(DTextEncoding::convertTextEncoding(dataUTF_32, convertedUTF8, "UTF-8"));
210 ASSERT_EQ(dataUTF_8, convertedUTF8);
211 }
212
213 TEST_F(ut_DTextEncoding, testConvertTextEncodingWithError)
214 {
215 QByteArray dataUTF_8;
216 ASSERT_FALSE(DTextEncoding::convertTextEncoding(dataGB18030, dataUTF_8, "ERROR"));
217 ASSERT_FALSE(DTextEncoding::convertTextEncoding(dataGB18030, dataUTF_8, "KOI8-R"));
218
219 QByteArray tmpUTF_8Error = "\x31\x32\x33\xFF\xFF\xFF\x31\x32\x33";
220 QByteArray tmpUTF_16;
221 QString error;
222 int converted = 0;
223 bool ret = DTextEncoding::convertTextEncodingEx(tmpUTF_8Error, tmpUTF_16, "UTF-16", "UTF-8", &error, &converted);
224 ASSERT_FALSE(ret);
225 ASSERT_FALSE(error.isEmpty());
226 ASSERT_EQ(converted, 3);
227
228 QByteArray tmpUTF_8Error2 = "\xFF\xFF";
229 converted = 3;
230 ret = DTextEncoding::convertTextEncodingEx(tmpUTF_8Error2, tmpUTF_16, "UTF-16", "UTF-8", &error, &converted);
231 ASSERT_FALSE(ret);
232 ASSERT_EQ(converted, 0);
233 }
234
235 TEST_F(ut_DTextEncoding, testConvertFileEncoding)
236 {
237 ASSERT_TRUE(rewriteTempFile(dataGB18030));
238 ASSERT_TRUE(DTextEncoding::convertFileEncoding(tmpFileName, "UTF-8"));
239 ASSERT_EQ("UTF-8", DTextEncoding::detectFileEncoding(tmpFileName));
240
241 ASSERT_TRUE(DTextEncoding::convertFileEncoding(tmpFileName, "UTF-32"));
242 ASSERT_EQ("UTF-32", DTextEncoding::detectFileEncoding(tmpFileName));
243
244 ASSERT_FALSE(DTextEncoding::convertFileEncoding("", "UTF-32"));
245 }
246
247 TEST_F(ut_DTextEncoding, testConvertFileEncodingTo)
248 {
249 QString tmpConvertFileName("/tmp/ut_DTextEncoding_temp_testConvertFileEncodingTo.txt");
250 if (QFile::exists(tmpConvertFileName)) {
251 ASSERT_TRUE(QFile::remove(tmpConvertFileName));
252 }
253
254 ASSERT_TRUE(rewriteTempFile(dataGB18030));
255 ASSERT_TRUE(DTextEncoding::convertFileEncodingTo(tmpFileName, tmpConvertFileName, "GB18030"));
256 ASSERT_TRUE(QFile::exists(tmpConvertFileName));
257
258 ASSERT_TRUE(DTextEncoding::convertFileEncodingTo(tmpFileName, tmpConvertFileName, "UTF-8"));
259 ASSERT_TRUE(QFile::exists(tmpConvertFileName));
260 ASSERT_EQ("UTF-8", DTextEncoding::detectFileEncoding(tmpConvertFileName));
261
262 ASSERT_TRUE(DTextEncoding::convertFileEncodingTo(tmpFileName, tmpConvertFileName, "UTF-32"));
263 ASSERT_EQ("UTF-32", DTextEncoding::detectFileEncoding(tmpConvertFileName));
264
265 ASSERT_TRUE(QFile::remove(tmpConvertFileName));
266 }
267
268 TEST_F(ut_DTextEncoding, testConvertFileEncodingToWithError)
269 {
270 QString tmpConvertFileName("/tmp/ut_DTextEncoding_temp_testConvertFileEncodingToWithError.txt");
271 if (QFile::exists(tmpConvertFileName)) {
272 ASSERT_TRUE(QFile::remove(tmpConvertFileName));
273 }
274
275 ASSERT_FALSE(DTextEncoding::convertFileEncodingTo("", tmpConvertFileName, "UTF-32"));
276 ASSERT_FALSE(DTextEncoding::convertFileEncodingTo(tmpFileName, "", "UTF-32"));
277 ASSERT_FALSE(DTextEncoding::convertFileEncodingTo(tmpFileName, tmpConvertFileName, "ERROR"));
278 ASSERT_FALSE(QFile::exists(tmpConvertFileName));
279
280 ASSERT_TRUE(rewriteTempFile(dataGB18030));
281 ASSERT_FALSE(DTextEncoding::convertFileEncodingTo(tmpFileName, tmpConvertFileName, "EUC-JP"));
282 ASSERT_FALSE(QFile::exists(tmpConvertFileName));
283 }