Debian Codegraph @

qt5/tests/check_utf_conversion.cpp

raw
1 #include <QtCore/QScopedPointer>
2 #include <QtTest/QTest>
3
4 #include <poppler-private.h>
5
6 #include <cstring>
7 #include <cstdint> // for uint16_t
8
9 #include "GlobalParams.h"
10 #include "UnicodeTypeTable.h"
11 #include "UTF.h"
12
13 class TestUTFConversion : public QObject
14 {
15 Q_OBJECT
16 public:
17 explicit TestUTFConversion(QObject *parent = nullptr) : QObject(parent) { }
18 private Q_SLOTS:
19 void testUTF_data();
20 void testUTF();
21 void testUnicodeToAscii7();
22 void testUnicodeLittleEndian();
23 };
24
25 static bool compare(const Unicode *a, const char *b, int len)
26 {
27 for (int i = 0; i < len; i++) {
28 if (a[i] != (Unicode)b[i]) {
29 return false;
30 }
31 }
32
33 return true;
34 }
35
36 static bool compare(const Unicode *a, const uint16_t *b, int len)
37 {
38 for (int i = 0; i < len; i++) {
39 if (a[i] != b[i]) {
40 return false;
41 }
42 }
43
44 return true;
45 }
46
47 void TestUTFConversion::testUTF_data()
48 {
49 QTest::addColumn<QString>("s");
50
51 QTest::newRow("<empty>") << QString(QLatin1String(""));
52 QTest::newRow("a") << QStringLiteral("a");
53 QTest::newRow("abc") << QStringLiteral("abc");
54 QTest::newRow("Latin") << QStringLiteral("Vitrum edere possum; mihi non nocet");
55 QTest::newRow("Greek") << QStringLiteral("Μπορώ να φάω σπασμένα γυαλιά χωρίς να πάθω τίποτα");
56 QTest::newRow("Icelandic") << QStringLiteral("Ég get etið gler án þess að meiða mig");
57 QTest::newRow("Russian") << QStringLiteral("Я могу есть стекло, оно мне не вредит.");
58 QTest::newRow("Sanskrit") << QStringLiteral("काचं शक्नोम्यत्तुम् । नोपहिनस्ति माम् ॥");
59 QTest::newRow("Arabic") << QStringLiteral("أنا قادر على أكل الزجاج و هذا لا يؤلمني");
60 QTest::newRow("Chinese") << QStringLiteral("我能吞下玻璃而不伤身体。");
61 QTest::newRow("Thai") << QStringLiteral("ฉันกินกระจกได้ แต่มันไม่ทำให้ฉันเจ็บ");
62 QTest::newRow("non BMP") << QStringLiteral("𝓹𝓸𝓹𝓹𝓵𝓮𝓻");
63 }
64
65 void TestUTFConversion::testUTF()
66 {
67 std::string utf8String;
68 int len;
69
70 QFETCH(QString, s);
71 const std::string str = s.toStdString();
72
73 // UTF-8 to UTF-16
74
75 len = utf8CountUtf16CodeUnits(str);
76 QCOMPARE(len, s.size()); // QString size() returns number of code units, not code points
77
78 std::u16string utf16String = utf8ToUtf16(str);
79 QCOMPARE(utf16String, s.toStdU16String());
80 QCOMPARE(len, s.size());
81
82 std::string gsUtf16_a(utf8ToUtf16WithBom(str));
83 std::unique_ptr<GooString> gsUtf16_b(Poppler::QStringToUnicodeGooString(s));
84 QCOMPARE(gsUtf16_b->cmp(gsUtf16_a), 0);
85
86 // UTF-16 to UTF-8
87
88 len = utf16CountUtf8Bytes(s.utf16());
89 QCOMPARE(len, str.size());
90
91 utf8String = utf16ToUtf8(s.utf16(), INT_MAX);
92 QCOMPARE(utf8String, str);
93 QCOMPARE(len, str.size());
94
95 utf8String = utf16ToUtf8(s.utf16());
96 QCOMPARE(utf8String, str);
97 }
98
99 void TestUTFConversion::testUnicodeToAscii7()
100 {
101 globalParams = std::make_unique<GlobalParams>();
102
103 // Test string is one 'Registered' and twenty 'Copyright' chars
104 // so it's long enough to reproduce the bug given that glibc
105 // malloc() always returns 8-byte aligned memory addresses.
106 std::unique_ptr<GooString> goo = Poppler::QStringToUnicodeGooString(QString::fromUtf8("®©©©©©©©©©©©©©©©©©©©©")); // clazy:exclude=qstring-allocations
107
108 const std::vector<Unicode> in = TextStringToUCS4(goo->toStr());
109
110 int in_norm_len;
111 int *in_norm_idx;
112 Unicode *in_norm = unicodeNormalizeNFKC(in.data(), in.size(), &in_norm_len, &in_norm_idx, true);
113
114 Unicode *out;
115 int out_len;
116 int *out_ascii_idx;
117
118 unicodeToAscii7(std::span(in_norm, in_norm_len), &out, &out_len, in_norm_idx, &out_ascii_idx);
119
120 free(in_norm);
121 free(in_norm_idx);
122
123 // ascii7 conversion: ® -> (R) © -> (c)
124 const char *expected_ascii = (char *)"(R)(c)(c)(c)(c)(c)(c)(c)(c)(c)(c)(c)(c)(c)(c)(c)(c)(c)(c)(c)(c)";
125
126 QCOMPARE(out_len, (int)strlen(expected_ascii));
127 QVERIFY(compare(out, expected_ascii, out_len));
128
129 free(out);
130 free(out_ascii_idx);
131 }
132
133 void TestUTFConversion::testUnicodeLittleEndian()
134 {
135 uint16_t UTF16LE_hi[5] { 0xFFFE, 0x4800, 0x4900, 0x2100, 0x1126 }; // UTF16-LE "HI!☑"
136 std::string GooUTF16LE(reinterpret_cast<const char *>(UTF16LE_hi), sizeof(UTF16LE_hi));
137
138 uint16_t UTF16BE_hi[5] { 0xFEFF, 0x0048, 0x0049, 0x0021, 0x2611 }; // UTF16-BE "HI!☑"
139 std::string GooUTF16BE(reinterpret_cast<const char *>(UTF16BE_hi), sizeof(UTF16BE_hi));
140
141 // Let's assert both GooString's are different
142 QVERIFY(GooUTF16LE != GooUTF16BE);
143
144 const std::vector<Unicode> UCS4fromLE = TextStringToUCS4(GooUTF16LE);
145 const std::vector<Unicode> UCS4fromBE = TextStringToUCS4(GooUTF16BE);
146
147 // len is 4 because TextStringToUCS4() removes the two leading Byte Order Mark (BOM) code points
148 QCOMPARE(UCS4fromLE.size(), UCS4fromBE.size());
149 QCOMPARE(UCS4fromLE.size(), 4);
150
151 // Check that now after conversion, UCS4fromLE and UCS4fromBE are now the same
152 for (size_t i = 0; i < UCS4fromLE.size(); i++) {
153 QCOMPARE(UCS4fromLE[i], UCS4fromBE[i]);
154 }
155
156 const QString expected = QString::fromUtf8("HI!☑"); // clazy:exclude=qstring-allocations
157
158 // Do some final verifications, checking the strings to be "HI!"
159 QVERIFY(UCS4fromLE == UCS4fromBE);
160 QVERIFY(compare(UCS4fromLE.data(), expected.utf16(), UCS4fromLE.size()));
161 QVERIFY(compare(UCS4fromBE.data(), expected.utf16(), UCS4fromLE.size()));
162 }
163
164 QTEST_GUILESS_MAIN(TestUTFConversion)
165 #include "check_utf_conversion.moc"