Debian Codegraph @

src/test/java/io/airlift/slice/TestSliceUtf8.java

raw
1 /*
2 * Licensed under the Apache License, Version 2.0 (the "License");
3 * you may not use this file except in compliance with the License.
4 * You may obtain a copy of the License at
5 *
6 * http://www.apache.org/licenses/LICENSE-2.0
7 *
8 * Unless required by applicable law or agreed to in writing, software
9 * distributed under the License is distributed on an "AS IS" BASIS,
10 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 * See the License for the specific language governing permissions and
12 * limitations under the License.
13 */
14 package io.airlift.slice;
15
16 import com.google.common.collect.ImmutableList;
17 import com.google.common.collect.Lists;
18 import com.google.common.primitives.Ints;
19 import org.testng.annotations.Test;
20
21 import java.util.Arrays;
22 import java.util.Collections;
23 import java.util.List;
24 import java.util.stream.IntStream;
25
26 import static com.google.common.primitives.Bytes.concat;
27 import static io.airlift.slice.SliceUtf8.codePointToUtf8;
28 import static io.airlift.slice.SliceUtf8.countCodePoints;
29 import static io.airlift.slice.SliceUtf8.fixInvalidUtf8;
30 import static io.airlift.slice.SliceUtf8.getCodePointAt;
31 import static io.airlift.slice.SliceUtf8.getCodePointBefore;
32 import static io.airlift.slice.SliceUtf8.isAscii;
33 import static io.airlift.slice.SliceUtf8.leftTrim;
34 import static io.airlift.slice.SliceUtf8.lengthOfCodePoint;
35 import static io.airlift.slice.SliceUtf8.lengthOfCodePointFromStartByte;
36 import static io.airlift.slice.SliceUtf8.lengthOfCodePointSafe;
37 import static io.airlift.slice.SliceUtf8.offsetOfCodePoint;
38 import static io.airlift.slice.SliceUtf8.reverse;
39 import static io.airlift.slice.SliceUtf8.rightTrim;
40 import static io.airlift.slice.SliceUtf8.setCodePointAt;
41 import static io.airlift.slice.SliceUtf8.substring;
42 import static io.airlift.slice.SliceUtf8.toLowerCase;
43 import static io.airlift.slice.SliceUtf8.toUpperCase;
44 import static io.airlift.slice.SliceUtf8.trim;
45 import static io.airlift.slice.Slices.EMPTY_SLICE;
46 import static io.airlift.slice.Slices.utf8Slice;
47 import static io.airlift.slice.Slices.wrappedBuffer;
48 import static java.lang.Character.MAX_CODE_POINT;
49 import static java.lang.Character.MIN_SURROGATE;
50 import static java.lang.Character.SURROGATE;
51 import static java.lang.Character.getType;
52 import static java.nio.charset.StandardCharsets.UTF_8;
53 import static org.testng.Assert.assertEquals;
54 import static org.testng.Assert.assertFalse;
55 import static org.testng.Assert.assertTrue;
56
57 public class TestSliceUtf8
58 {
59 private static final int[] ASCII_CODE_POINTS;
60 private static final String STRING_ASCII_CODE_POINTS;
61 private static final int[] ALL_CODE_POINTS;
62 private static final String STRING_ALL_CODE_POINTS;
63 private static final int[] ALL_CODE_POINTS_RANDOM;
64 private static final String STRING_ALL_CODE_POINTS_RANDOM;
65
66 private static final byte START_1_BYTE = (byte) 0b0111_1111;
67 private static final byte CONTINUATION_BYTE = (byte) 0b1011_1111;
68 private static final byte START_2_BYTE = (byte) 0b1101_1111;
69 private static final byte START_3_BYTE = (byte) 0b1110_1111;
70 private static final byte START_4_BYTE = (byte) 0b1111_0111;
71 private static final byte START_5_BYTE = (byte) 0b1111_1011;
72 private static final byte START_6_BYTE = (byte) 0b1111_1101;
73 private static final byte INVALID_FE_BYTE = (byte) 0b11111110;
74 private static final byte INVALID_FF_BYTE = (byte) 0b11111111;
75 private static final byte X_CHAR = (byte) 'X';
76
77 private static final List<byte[]> INVALID_SEQUENCES;
78
79 static {
80 ASCII_CODE_POINTS = IntStream.rangeClosed(0, 0x7F)
81 .toArray();
82 STRING_ASCII_CODE_POINTS = new String(ASCII_CODE_POINTS, 0, ASCII_CODE_POINTS.length);
83
84 ALL_CODE_POINTS = IntStream.rangeClosed(0, MAX_CODE_POINT)
85 .filter(codePoint -> getType(codePoint) != SURROGATE)
86 .toArray();
87 STRING_ALL_CODE_POINTS = new String(ALL_CODE_POINTS, 0, ALL_CODE_POINTS.length);
88
89 ALL_CODE_POINTS_RANDOM = Arrays.copyOf(ALL_CODE_POINTS, ALL_CODE_POINTS.length);
90 Collections.shuffle(Arrays.asList(ALL_CODE_POINTS_RANDOM));
91 STRING_ALL_CODE_POINTS_RANDOM = new String(ALL_CODE_POINTS_RANDOM, 0, ALL_CODE_POINTS_RANDOM.length);
92
93 ImmutableList.Builder<byte[]> invalidSequences = ImmutableList.builder();
94 invalidSequences.add(new byte[] {CONTINUATION_BYTE});
95 invalidSequences.add(new byte[] {START_2_BYTE});
96 invalidSequences.add(new byte[] {START_3_BYTE});
97 invalidSequences.add(new byte[] {START_3_BYTE, CONTINUATION_BYTE});
98 invalidSequences.add(new byte[] {START_4_BYTE});
99 invalidSequences.add(new byte[] {START_4_BYTE, CONTINUATION_BYTE});
100 invalidSequences.add(new byte[] {START_4_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE});
101 // 4 byte sequence is limited to 10FFFF
102 invalidSequences.add(new byte[] {START_4_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE});
103 invalidSequences.add(new byte[] {START_5_BYTE});
104 invalidSequences.add(new byte[] {START_5_BYTE, CONTINUATION_BYTE});
105 invalidSequences.add(new byte[] {START_5_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE});
106 invalidSequences.add(new byte[] {START_5_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE});
107 invalidSequences.add(new byte[] {START_5_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE});
108 invalidSequences.add(new byte[] {START_6_BYTE});
109 invalidSequences.add(new byte[] {START_6_BYTE, CONTINUATION_BYTE});
110 invalidSequences.add(new byte[] {START_6_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE});
111 invalidSequences.add(new byte[] {START_6_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE});
112 invalidSequences.add(new byte[] {START_6_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE});
113 invalidSequences.add(new byte[] {START_6_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE});
114 invalidSequences.add(new byte[] {INVALID_FF_BYTE});
115
116 // min and max surrogate characters
117 invalidSequences.add(new byte[] {(byte) 0b11101101, (byte) 0xA0, (byte) 0x80});
118 invalidSequences.add(new byte[] {(byte) 0b11101101, (byte) 0xBF, (byte) 0xBF});
119 INVALID_SEQUENCES = invalidSequences.build();
120 }
121
122 private static final String STRING_EMPTY = "";
123 private static final String STRING_HELLO = "hello";
124 private static final String STRING_QUADRATICALLY = "Quadratically";
125 private static final String STRING_OESTERREICH = "\u00D6sterreich";
126 private static final String STRING_DULIOE_DULIOE = "Duli\u00F6 duli\u00F6";
127 private static final String STRING_FAITH_HOPE_LOVE = "\u4FE1\u5FF5,\u7231,\u5E0C\u671B";
128 private static final String STRING_NAIVE = "na\u00EFve";
129 private static final String STRING_OO = "\uD801\uDC2Dend";
130 // length increase when cast to lower case, and ends with invalid character
131 private static final byte[] INVALID_SEQUENCE_TO_LOWER_EXPANDS = new byte[] {(byte) 0xC8, (byte) 0xBA, (byte) 0xFF};
132
133 private static final byte[] INVALID_UTF8_1 = new byte[] {-127};
134 private static final byte[] INVALID_UTF8_2 = new byte[] {50, -127, 52, 50};
135
136 @Test
137 public void testCodePointCount()
138 {
139 assertCodePointCount(STRING_EMPTY);
140 assertCodePointCount(STRING_HELLO);
141 assertCodePointCount(STRING_QUADRATICALLY);
142 assertCodePointCount(STRING_OESTERREICH);
143 assertCodePointCount(STRING_DULIOE_DULIOE);
144 assertCodePointCount(STRING_FAITH_HOPE_LOVE);
145 assertCodePointCount(STRING_NAIVE);
146 assertCodePointCount(STRING_OO);
147 assertCodePointCount(STRING_ASCII_CODE_POINTS);
148 assertCodePointCount(STRING_ALL_CODE_POINTS);
149 assertCodePointCount(STRING_ALL_CODE_POINTS_RANDOM);
150
151 assertEquals(countCodePoints(wrappedBuffer(START_1_BYTE)), 1);
152 assertEquals(countCodePoints(wrappedBuffer(START_2_BYTE)), 1);
153 assertEquals(countCodePoints(wrappedBuffer(START_3_BYTE)), 1);
154 assertEquals(countCodePoints(wrappedBuffer(START_4_BYTE)), 1);
155 assertEquals(countCodePoints(wrappedBuffer(START_5_BYTE)), 1);
156 assertEquals(countCodePoints(wrappedBuffer(START_6_BYTE)), 1);
157 assertEquals(countCodePoints(wrappedBuffer(INVALID_FE_BYTE)), 1);
158 assertEquals(countCodePoints(wrappedBuffer(INVALID_FF_BYTE)), 1);
159 assertEquals(countCodePoints(wrappedBuffer(CONTINUATION_BYTE)), 0);
160 }
161
162 private static void assertCodePointCount(String string)
163 {
164 assertEquals(countCodePoints(utf8Slice(string)), string.codePoints().count());
165 }
166
167 @Test
168 public void testOffsetByCodePoints()
169 {
170 assertEquals(offsetOfCodePoint(EMPTY_SLICE, 0), -1);
171 assertOffsetByCodePoints(STRING_HELLO);
172 assertOffsetByCodePoints(STRING_QUADRATICALLY);
173 assertOffsetByCodePoints(STRING_OESTERREICH);
174 assertOffsetByCodePoints(STRING_DULIOE_DULIOE);
175 assertOffsetByCodePoints(STRING_FAITH_HOPE_LOVE);
176 assertOffsetByCodePoints(STRING_NAIVE);
177 assertOffsetByCodePoints(STRING_OO);
178 assertOffsetByCodePoints(STRING_ASCII_CODE_POINTS);
179 assertOffsetByCodePoints(STRING_ALL_CODE_POINTS);
180 assertOffsetByCodePoints(STRING_ALL_CODE_POINTS_RANDOM);
181 }
182
183 private static void assertOffsetByCodePoints(String string)
184 {
185 Slice utf8 = utf8Slice(string);
186
187 int codePoints = (int) string.codePoints().count();
188 int lastIndex = 0;
189 int characterIndex = 0;
190 for (int codePointIndex = 0; codePointIndex < codePoints; codePointIndex++) {
191 int expectedIndex = 0;
192
193 // calculate the expected index by searching forward from the last index
194 if (codePointIndex > 0) {
195 expectedIndex = lastIndex + lengthOfCodePoint(string.codePointAt(characterIndex));
196 characterIndex = string.offsetByCodePoints(characterIndex, 1);
197 }
198 // avoid n^2 performance for large test string
199 if (codePointIndex < 10000) {
200 assertEquals(offsetOfCodePoint(utf8, codePointIndex), expectedIndex);
201 }
202
203 if (codePointIndex > 0) {
204 assertEquals(offsetOfCodePoint(utf8, lastIndex, 1), expectedIndex);
205 }
206 lastIndex = expectedIndex;
207 }
208 assertEquals(offsetOfCodePoint(utf8Slice(string), codePoints), -1);
209 }
210
211 @Test
212 public void testSubstring()
213 {
214 assertSubstring(STRING_HELLO);
215 assertSubstring(STRING_QUADRATICALLY);
216 assertSubstring(STRING_OESTERREICH);
217 assertSubstring(STRING_DULIOE_DULIOE);
218 assertSubstring(STRING_FAITH_HOPE_LOVE);
219 assertSubstring(STRING_NAIVE);
220 assertSubstring(STRING_OO);
221 assertSubstring(STRING_ASCII_CODE_POINTS);
222 // substring test over all code points takes too long, so only run it on the tail
223 // that has the largest code points
224 assertSubstring(new String(ALL_CODE_POINTS, ALL_CODE_POINTS.length - 500, 500));
225 }
226
227 private static void assertSubstring(String string)
228 {
229 Slice utf8 = utf8Slice(string);
230
231 int[] codePoints = string.codePoints().toArray();
232 for (int start = 0; start < codePoints.length / 2; start++) {
233 int count = Math.min(20, codePoints.length - start - start - 1);
234 Slice actual = substring(utf8, start, count);
235 Slice expected = wrappedBuffer(new String(codePoints, start, count).getBytes(UTF_8));
236 assertEquals(actual, expected);
237 }
238 assertEquals(substring(utf8, 0, codePoints.length), utf8);
239 assertEquals(substring(utf8, 0, 0), EMPTY_SLICE);
240 }
241
242 @Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = "UTF-8 does not contain 10 code points")
243 public void testSubstringInvalidStart()
244 {
245 substring(utf8Slice(STRING_HELLO), 10, 2);
246 }
247
248 @Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = "UTF-8 does not contain 7 code points")
249 public void testSubstringInvalidLength()
250 {
251 substring(utf8Slice(STRING_HELLO), 0, 7);
252 }
253
254 @Test(expectedExceptions = InvalidUtf8Exception.class, expectedExceptionsMessageRegExp = "UTF-8 is not well formed")
255 public void testSubstringInvalidUtf8()
256 {
257 substring(wrappedBuffer((byte) 'f', (byte) 'o', (byte) 'o', START_3_BYTE, CONTINUATION_BYTE), 0, 4);
258 }
259
260 @Test
261 public void testReverse()
262 {
263 assertReverse(STRING_HELLO);
264 assertReverse(STRING_QUADRATICALLY);
265 assertReverse(STRING_OESTERREICH);
266 assertReverse(STRING_DULIOE_DULIOE);
267 assertReverse(STRING_FAITH_HOPE_LOVE);
268 assertReverse(STRING_NAIVE);
269 assertReverse(STRING_OO);
270 assertReverse(STRING_ASCII_CODE_POINTS);
271 assertReverse(STRING_ALL_CODE_POINTS);
272
273 INVALID_SEQUENCES.forEach(TestSliceUtf8::assertReverseWithInvalidSequence);
274 }
275
276 private static void assertReverse(String string)
277 {
278 Slice actualReverse = reverse(utf8Slice(string));
279
280 int[] codePoints = string.codePoints().toArray();
281 codePoints = Ints.toArray(Lists.reverse(Ints.asList(codePoints)));
282 Slice expectedReverse = wrappedBuffer(new String(codePoints, 0, codePoints.length).getBytes(UTF_8));
283
284 assertEquals(actualReverse, expectedReverse);
285 }
286
287 private static void assertReverseWithInvalidSequence(byte[] invalidSequence)
288 {
289 assertEquals(
290 reverse(wrappedBuffer(invalidSequence)),
291 wrappedBuffer(invalidSequence));
292 assertEquals(
293 reverse(wrappedBuffer(concat(new byte[] {'a', 'b', 'c'}, invalidSequence))),
294 wrappedBuffer(concat(invalidSequence, new byte[] {'c', 'b', 'a'})));
295 assertEquals(
296 reverse(wrappedBuffer(concat(invalidSequence, new byte[] {'x', 'y', 'z'}))),
297 wrappedBuffer(concat(new byte[] {'z', 'y', 'x'}, invalidSequence)));
298 assertEquals(
299 reverse(wrappedBuffer(concat(new byte[] {'a', 'b', 'c'}, invalidSequence, new byte[] {'x', 'y', 'z'}))),
300 wrappedBuffer(concat(new byte[] {'z', 'y', 'x'}, invalidSequence, new byte[] {'c', 'b', 'a'})));
301 }
302
303 @Test
304 public void testIsAscii()
305 {
306 assertTrue(isAscii(utf8Slice(STRING_HELLO)));
307 assertTrue(isAscii(utf8Slice(STRING_QUADRATICALLY)));
308 assertFalse(isAscii(utf8Slice(STRING_OESTERREICH)));
309 assertFalse(isAscii(utf8Slice(STRING_DULIOE_DULIOE)));
310 assertFalse(isAscii(utf8Slice(STRING_FAITH_HOPE_LOVE)));
311 assertFalse(isAscii(utf8Slice(STRING_NAIVE)));
312 assertFalse(isAscii(utf8Slice(STRING_OO)));
313 assertTrue(isAscii(utf8Slice(STRING_ASCII_CODE_POINTS)));
314 assertFalse(isAscii(utf8Slice(STRING_ALL_CODE_POINTS)));
315 }
316
317 @Test
318 public void testFixInvalidUtf8()
319 {
320 assertFixInvalidUtf8(utf8Slice(STRING_OESTERREICH), utf8Slice(STRING_OESTERREICH));
321 assertFixInvalidUtf8(utf8Slice(STRING_HELLO), utf8Slice(STRING_HELLO));
322 assertFixInvalidUtf8(utf8Slice(STRING_QUADRATICALLY), utf8Slice(STRING_QUADRATICALLY));
323 assertFixInvalidUtf8(utf8Slice(STRING_OESTERREICH), utf8Slice(STRING_OESTERREICH));
324 assertFixInvalidUtf8(utf8Slice(STRING_DULIOE_DULIOE), utf8Slice(STRING_DULIOE_DULIOE));
325 assertFixInvalidUtf8(utf8Slice(STRING_FAITH_HOPE_LOVE), utf8Slice(STRING_FAITH_HOPE_LOVE));
326 assertFixInvalidUtf8(utf8Slice(STRING_NAIVE), utf8Slice(STRING_NAIVE));
327 assertFixInvalidUtf8(utf8Slice(STRING_OO), utf8Slice(STRING_OO));
328 assertFixInvalidUtf8(utf8Slice(STRING_ASCII_CODE_POINTS), utf8Slice(STRING_ASCII_CODE_POINTS));
329 assertFixInvalidUtf8(utf8Slice(STRING_ALL_CODE_POINTS), utf8Slice(STRING_ALL_CODE_POINTS));
330
331 // max valid value for 2, 3, and 4 byte sequences
332 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_2_BYTE, CONTINUATION_BYTE), utf8Slice("X\u07FF"));
333 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_3_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE), utf8Slice("X\uFFFF"));
334 // 4 byte sequence is limited to U+10FFFF by RFC 3629
335 assertFixInvalidUtf8(
336 wrappedBuffer(X_CHAR, (byte) 0xF4, (byte) 0x8F, CONTINUATION_BYTE, CONTINUATION_BYTE),
337 wrappedBuffer(X_CHAR, (byte) 0xF4, (byte) 0x8F, CONTINUATION_BYTE, CONTINUATION_BYTE));
338
339 // 4 byte sequence is limited to 10FFFF
340 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_4_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE), utf8Slice("X\uFFFD"));
341
342 // 5 and 6 byte sequences are always invalid
343 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_5_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE), utf8Slice("X\uFFFD"));
344 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_6_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE), utf8Slice("X\uFFFD"));
345
346 // continuation byte alone is invalid
347 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, CONTINUATION_BYTE), utf8Slice("X\uFFFD"));
348
349 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, INVALID_FE_BYTE), utf8Slice("X\uFFFD"));
350 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, INVALID_FF_BYTE), utf8Slice("X\uFFFD"));
351
352 // sequences with not enough continuation bytes, but enough bytes
353 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_2_BYTE, X_CHAR), utf8Slice("X\uFFFDX"));
354 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_3_BYTE, X_CHAR, X_CHAR), utf8Slice("X\uFFFDXX"));
355 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_3_BYTE, CONTINUATION_BYTE, X_CHAR), utf8Slice("X\uFFFDX"));
356 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_4_BYTE, X_CHAR, X_CHAR, X_CHAR), utf8Slice("X\uFFFDXXX"));
357 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_4_BYTE, CONTINUATION_BYTE, X_CHAR, X_CHAR), utf8Slice("X\uFFFDXX"));
358 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_4_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, X_CHAR), utf8Slice("X\uFFFDX"));
359 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_5_BYTE, X_CHAR, X_CHAR, X_CHAR, X_CHAR), utf8Slice("X\uFFFDXXXX"));
360 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_5_BYTE, CONTINUATION_BYTE, X_CHAR, X_CHAR, X_CHAR), utf8Slice("X\uFFFDXXX"));
361 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_5_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, X_CHAR, X_CHAR), utf8Slice("X\uFFFDXX"));
362 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_5_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, X_CHAR), utf8Slice("X\uFFFDX"));
363 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_6_BYTE, X_CHAR, X_CHAR, X_CHAR, X_CHAR, X_CHAR), utf8Slice("X\uFFFDXXXXX"));
364 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_6_BYTE, CONTINUATION_BYTE, X_CHAR, X_CHAR, X_CHAR, X_CHAR), utf8Slice("X\uFFFDXXXX"));
365 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_6_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, X_CHAR, X_CHAR, X_CHAR), utf8Slice("X\uFFFDXXX"));
366 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_6_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, X_CHAR, X_CHAR), utf8Slice("X\uFFFDXX"));
367 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_6_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, X_CHAR), utf8Slice("X\uFFFDX"));
368 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, INVALID_FE_BYTE), utf8Slice("X\uFFFD"));
369 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, INVALID_FF_BYTE), utf8Slice("X\uFFFD"));
370
371 // truncated sequences
372 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, CONTINUATION_BYTE), utf8Slice("X\uFFFD"));
373 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_2_BYTE), utf8Slice("X\uFFFD"));
374 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_3_BYTE), utf8Slice("X\uFFFD"));
375 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_3_BYTE, CONTINUATION_BYTE), utf8Slice("X\uFFFD"));
376 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_4_BYTE), utf8Slice("X\uFFFD"));
377 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_4_BYTE, CONTINUATION_BYTE), utf8Slice("X\uFFFD"));
378 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_4_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE), utf8Slice("X\uFFFD"));
379 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_5_BYTE), utf8Slice("X\uFFFD"));
380 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_5_BYTE, CONTINUATION_BYTE), utf8Slice("X\uFFFD"));
381 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_5_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE), utf8Slice("X\uFFFD"));
382 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_5_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE), utf8Slice("X\uFFFD"));
383 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_6_BYTE), utf8Slice("X\uFFFD"));
384 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_6_BYTE, CONTINUATION_BYTE), utf8Slice("X\uFFFD"));
385 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_6_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE), utf8Slice("X\uFFFD"));
386 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_6_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE), utf8Slice("X\uFFFD"));
387 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_6_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE), utf8Slice("X\uFFFD"));
388 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, START_6_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE), utf8Slice("X\uFFFD"));
389 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, INVALID_FE_BYTE), utf8Slice("X\uFFFD"));
390 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, INVALID_FF_BYTE), utf8Slice("X\uFFFD"));
391 // min and max surrogate characters
392 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, (byte) 0b11101101, (byte) 0xA0, (byte) 0x80), utf8Slice("X\uFFFD"));
393 assertFixInvalidUtf8(wrappedBuffer(X_CHAR, (byte) 0b11101101, (byte) 0xBF, (byte) 0xBF), utf8Slice("X\uFFFD"));
394 }
395
396 private static void assertFixInvalidUtf8(Slice testSlice, Slice expectedSlice)
397 {
398 assertEquals(fixInvalidUtf8(testSlice), expectedSlice);
399 }
400
401 @Test
402 public void testCaseChange()
403 {
404 assertCaseChange(STRING_ALL_CODE_POINTS);
405 assertCaseChange(STRING_FAITH_HOPE_LOVE);
406 assertCaseChange(STRING_HELLO);
407 assertCaseChange(STRING_QUADRATICALLY);
408 assertCaseChange(STRING_OESTERREICH);
409 assertCaseChange(STRING_DULIOE_DULIOE);
410 assertCaseChange(STRING_FAITH_HOPE_LOVE);
411 assertCaseChange(STRING_NAIVE);
412 assertCaseChange(STRING_OO);
413 assertCaseChange(STRING_ASCII_CODE_POINTS);
414 assertCaseChange(STRING_ALL_CODE_POINTS);
415 assertCaseChange(STRING_ALL_CODE_POINTS_RANDOM);
416
417 toLowerCase(Slices.wrappedBuffer(INVALID_SEQUENCE_TO_LOWER_EXPANDS));
418
419 INVALID_SEQUENCES.forEach(TestSliceUtf8::assertCaseChangeWithInvalidSequence);
420 }
421
422 private static void assertCaseChangeWithInvalidSequence(byte[] invalidSequence)
423 {
424 assertEquals(
425 toLowerCase(wrappedBuffer(invalidSequence)),
426 wrappedBuffer(invalidSequence));
427 assertEquals(
428 toUpperCase(wrappedBuffer(invalidSequence)),
429 wrappedBuffer(invalidSequence));
430
431 assertEquals(
432 toLowerCase(wrappedBuffer(concat(new byte[] {'F', 'O', 'O'}, invalidSequence))),
433 wrappedBuffer(concat(new byte[] {'f', 'o', 'o'}, invalidSequence)));
434 assertEquals(
435 toUpperCase(wrappedBuffer(concat(new byte[] {'f', 'o', 'o'}, invalidSequence))),
436 wrappedBuffer(concat(new byte[] {'F', 'O', 'O'}, invalidSequence)));
437
438 assertEquals(
439 toLowerCase(wrappedBuffer(concat(invalidSequence, new byte[] {'F', 'O', 'O'}))),
440 wrappedBuffer(concat(invalidSequence, new byte[] {'f', 'o', 'o'})));
441 assertEquals(
442 toUpperCase(wrappedBuffer(concat(invalidSequence, new byte[] {'f', 'o', 'o'}))),
443 wrappedBuffer(concat(invalidSequence, new byte[] {'F', 'O', 'O'})));
444
445 assertEquals(
446 toLowerCase(wrappedBuffer(concat(new byte[] {'F', 'O', 'O'}, invalidSequence, new byte[] {'B', 'A', 'R'}))),
447 wrappedBuffer(concat(new byte[] {'f', 'o', 'o'}, invalidSequence, new byte[] {'b', 'a', 'r'})));
448 assertEquals(
449 toUpperCase(wrappedBuffer(concat(new byte[] {'f', 'o', 'o'}, invalidSequence, new byte[] {'b', 'a', 'r'}))),
450 wrappedBuffer(concat(new byte[] {'F', 'O', 'O'}, invalidSequence, new byte[] {'B', 'A', 'R'})));
451
452 }
453
454 private static void assertCaseChange(String string)
455 {
456 String expectedLower = lowerByCodePoint(string);
457 Slice actualLower = toLowerCase(utf8Slice(string));
458 assertEquals(actualLower, wrappedBuffer(expectedLower.getBytes(UTF_8)));
459
460 String expectedUpper = upperByCodePoint(string);
461 Slice actualUpper = toUpperCase(utf8Slice(string));
462 assertEquals(actualUpper, wrappedBuffer(expectedUpper.getBytes(UTF_8)));
463
464 // lower the upper and upper the lower
465 // NOTE: not all code points roundtrip, so calculate the expected
466 assertEquals(toLowerCase(actualUpper), wrappedBuffer(lowerByCodePoint(expectedUpper).getBytes(UTF_8)));
467 assertEquals(toUpperCase(actualLower), wrappedBuffer(upperByCodePoint(expectedLower).getBytes(UTF_8)));
468 }
469
470 private static String lowerByCodePoint(String string)
471 {
472 int[] upperCodePoints = string.codePoints().map(Character::toLowerCase).toArray();
473 return new String(upperCodePoints, 0, upperCodePoints.length);
474 }
475
476 private static String upperByCodePoint(String string)
477 {
478 int[] upperCodePoints = string.codePoints().map(Character::toUpperCase).toArray();
479 return new String(upperCodePoints, 0, upperCodePoints.length);
480 }
481
482 @Test
483 public void testLeftTrim()
484 {
485 assertLeftTrim("");
486 assertLeftTrim("hello");
487 assertLeftTrim("hello world");
488 assertLeftTrim("hello world ");
489
490 INVALID_SEQUENCES.forEach(TestSliceUtf8::assertLeftTrim);
491 }
492
493 private static void assertLeftTrim(String string)
494 {
495 assertLeftTrim(string.getBytes(UTF_8));
496 }
497
498 private static void assertLeftTrim(byte[] sequence)
499 {
500 assertEquals(leftTrim(wrappedBuffer(sequence)), wrappedBuffer(sequence));
501 for (int codePoint : ALL_CODE_POINTS) {
502 if (Character.isWhitespace(codePoint)) {
503 byte[] whitespace = new String(new int[] {codePoint}, 0, 1).getBytes(UTF_8);
504 assertEquals(leftTrim(wrappedBuffer(concat(whitespace, sequence))), wrappedBuffer(sequence));
505 assertEquals(leftTrim(wrappedBuffer(concat(whitespace, new byte[] {'\r', '\n', '\t', ' '}, whitespace, sequence))), wrappedBuffer(sequence));
506 }
507 }
508 }
509
510 @Test
511 public void testRightTrim()
512 {
513 assertRightTrim("");
514 assertRightTrim("hello");
515 assertRightTrim("hello world");
516 assertRightTrim(" hello world");
517
518 INVALID_SEQUENCES.forEach(TestSliceUtf8::assertRightTrim);
519 }
520
521 private static void assertRightTrim(String string)
522 {
523 assertRightTrim(string.getBytes(UTF_8));
524 }
525
526 private static void assertRightTrim(byte[] sequence)
527 {
528 assertEquals(rightTrim(wrappedBuffer(sequence)), wrappedBuffer(sequence));
529 for (int codePoint : ALL_CODE_POINTS) {
530 if (Character.isWhitespace(codePoint)) {
531 byte[] whitespace = new String(new int[] {codePoint}, 0, 1).getBytes(UTF_8);
532 assertEquals(rightTrim(wrappedBuffer(concat(sequence, whitespace))), wrappedBuffer(sequence));
533 assertEquals(rightTrim(wrappedBuffer(concat(sequence, whitespace, new byte[] {'\r', '\n', '\t', ' '}, whitespace))), wrappedBuffer(sequence));
534 }
535 }
536 }
537
538 @Test
539 public void testTrim()
540 {
541 assertTrim("");
542 assertTrim("hello");
543 assertTrim("hello world");
544
545 INVALID_SEQUENCES.forEach(TestSliceUtf8::assertTrim);
546 }
547
548 private static void assertTrim(String string)
549 {
550 assertTrim(string.getBytes(UTF_8));
551 }
552
553 private static void assertTrim(byte[] sequence)
554 {
555 assertEquals(trim(wrappedBuffer(sequence)), wrappedBuffer(sequence));
556 for (int codePoint : ALL_CODE_POINTS) {
557 if (Character.isWhitespace(codePoint)) {
558 byte[] whitespace = new String(new int[] {codePoint}, 0, 1).getBytes(UTF_8);
559 assertEquals(trim(wrappedBuffer(concat(whitespace, sequence, whitespace))), wrappedBuffer(sequence));
560 assertEquals(
561 trim(wrappedBuffer(concat(whitespace, new byte[] {'\r', '\n', '\t', ' '}, whitespace, sequence, whitespace, new byte[] {'\r', '\n', '\t', ' '}, whitespace))),
562 wrappedBuffer(sequence));
563 }
564 }
565 }
566
567 /**
568 * Test invalid UTF8 encodings. We do not expect a 'correct' but none harmful result.
569 */
570 @Test
571 public void testInvalidUtf8()
572 {
573 assertEquals(countCodePoints(wrappedBuffer(INVALID_UTF8_1)), 0);
574 assertEquals(countCodePoints(wrappedBuffer(INVALID_UTF8_2)), 3);
575
576 assertEquals(offsetOfCodePoint(wrappedBuffer(INVALID_UTF8_1), 0), 0);
577 assertEquals(offsetOfCodePoint(wrappedBuffer(INVALID_UTF8_1), 1), -1);
578
579 assertEquals(offsetOfCodePoint(wrappedBuffer(INVALID_UTF8_2), 0), 0);
580 assertEquals(offsetOfCodePoint(wrappedBuffer(INVALID_UTF8_2), 1), 2);
581 assertEquals(offsetOfCodePoint(wrappedBuffer(INVALID_UTF8_2), 2), 3);
582 assertEquals(offsetOfCodePoint(wrappedBuffer(INVALID_UTF8_2), 3), -1);
583 }
584
585 @Test
586 public void testLengthOfCodePoint()
587 {
588 assertEquals(lengthOfCodePointFromStartByte(START_1_BYTE), 1);
589 assertEquals(lengthOfCodePointFromStartByte(START_2_BYTE), 2);
590 assertEquals(lengthOfCodePointFromStartByte(START_3_BYTE), 3);
591 assertEquals(lengthOfCodePointFromStartByte(START_4_BYTE), 4);
592
593 for (int codePoint : ALL_CODE_POINTS) {
594 String string = new String(new int[] {codePoint}, 0, 1);
595 assertEquals(string.codePoints().count(), 1);
596
597 Slice utf8 = wrappedBuffer(string.getBytes(UTF_8));
598 assertEquals(lengthOfCodePoint(codePoint), utf8.length());
599 assertEquals(lengthOfCodePoint(utf8, 0), utf8.length());
600 assertEquals(lengthOfCodePointSafe(utf8, 0), utf8.length());
601 assertEquals(lengthOfCodePointFromStartByte(utf8.getByte(0)), utf8.length());
602
603 assertEquals(getCodePointAt(utf8, 0), codePoint);
604 assertEquals(getCodePointBefore(utf8, utf8.length()), codePoint);
605
606 assertEquals(codePointToUtf8(codePoint), utf8);
607 }
608
609 for (byte[] sequence : INVALID_SEQUENCES) {
610 assertEquals(lengthOfCodePointSafe(wrappedBuffer(sequence), 0), sequence.length);
611 assertEquals(lengthOfCodePointSafe(wrappedBuffer(concat(new byte[] {'x'}, sequence)), 1), sequence.length);
612 assertEquals(lengthOfCodePointSafe(wrappedBuffer(concat(sequence, new byte[] {'x'})), 0), sequence.length);
613 }
614 }
615
616 @Test(expectedExceptions = InvalidCodePointException.class, expectedExceptionsMessageRegExp = "Invalid code point 0xFFFFFFFF")
617 public void testLengthOfNegativeCodePoint()
618 {
619 lengthOfCodePoint(-1);
620 }
621
622 @Test(expectedExceptions = InvalidCodePointException.class, expectedExceptionsMessageRegExp = "Invalid code point 0x110000")
623 public void testLengthOfOutOfRangeCodePoint()
624 {
625 lengthOfCodePoint(MAX_CODE_POINT + 1);
626 }
627
628 @Test(expectedExceptions = InvalidUtf8Exception.class, expectedExceptionsMessageRegExp = "Illegal start 0xBF of code point")
629 public void testLengthOfCodePointContinuationByte()
630 {
631 lengthOfCodePointFromStartByte(CONTINUATION_BYTE);
632 }
633
634 @Test(expectedExceptions = InvalidUtf8Exception.class, expectedExceptionsMessageRegExp = "Illegal start 0xFB of code point")
635 public void testLengthOfCodePoint5ByteSequence()
636 {
637 lengthOfCodePointFromStartByte(START_5_BYTE);
638 }
639
640 @Test(expectedExceptions = InvalidUtf8Exception.class, expectedExceptionsMessageRegExp = "Illegal start 0xFD of code point")
641 public void testLengthOfCodePoint6ByteByte()
642 {
643 lengthOfCodePointFromStartByte(START_6_BYTE);
644 }
645
646 @Test(expectedExceptions = InvalidUtf8Exception.class, expectedExceptionsMessageRegExp = "Illegal start 0xFE of code point")
647 public void testLengthOfCodePointFEByte()
648 {
649 lengthOfCodePointFromStartByte(INVALID_FE_BYTE);
650 }
651
652 @Test(expectedExceptions = InvalidUtf8Exception.class, expectedExceptionsMessageRegExp = "Illegal start 0xFF of code point")
653 public void testLengthOfCodePointFFByte()
654 {
655 lengthOfCodePointFromStartByte(INVALID_FF_BYTE);
656 }
657
658 @Test(expectedExceptions = InvalidUtf8Exception.class, expectedExceptionsMessageRegExp = "UTF-8 sequence truncated")
659 public void testCodePointAtTruncated2()
660 {
661 getCodePointAt(wrappedBuffer((byte) 'x', START_2_BYTE), 1);
662 }
663
664 @Test(expectedExceptions = InvalidUtf8Exception.class, expectedExceptionsMessageRegExp = "UTF-8 sequence truncated")
665 public void testCodePointAtTruncated3()
666 {
667 getCodePointAt(wrappedBuffer((byte) 'x', START_3_BYTE, CONTINUATION_BYTE), 1);
668 }
669
670 @Test(expectedExceptions = InvalidUtf8Exception.class, expectedExceptionsMessageRegExp = "UTF-8 sequence truncated")
671 public void testCodePointAtTruncated4()
672 {
673 getCodePointAt(wrappedBuffer((byte) 'x', START_4_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE), 1);
674 }
675
676 @Test(expectedExceptions = InvalidUtf8Exception.class, expectedExceptionsMessageRegExp = "Illegal start 0xFB of code point")
677 public void testCodePointAt5ByteSequence()
678 {
679 getCodePointAt(wrappedBuffer((byte) 'x', START_5_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE), 1);
680 }
681
682 @Test(expectedExceptions = InvalidUtf8Exception.class, expectedExceptionsMessageRegExp = "UTF-8 is not well formed")
683 public void testCodePointBefore5ByteSequence()
684 {
685 getCodePointBefore(wrappedBuffer((byte) 'x', START_5_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE, CONTINUATION_BYTE), 6);
686 }
687
688 @Test(expectedExceptions = InvalidCodePointException.class, expectedExceptionsMessageRegExp = "Invalid code point 0xFFFFFFFF")
689 public void testSetNegativeCodePoint()
690 {
691 setCodePointAt(-1, Slices.allocate(8), 0);
692 }
693 @Test(expectedExceptions = InvalidCodePointException.class, expectedExceptionsMessageRegExp = "Invalid code point 0xD800")
694 public void testSetSurrogateCodePoint()
695 {
696 setCodePointAt(MIN_SURROGATE, Slices.allocate(8), 0);
697 }
698
699 @Test(expectedExceptions = InvalidCodePointException.class, expectedExceptionsMessageRegExp = "Invalid code point 0x110000")
700 public void testSetOutOfRangeCodePoint()
701 {
702 setCodePointAt(MAX_CODE_POINT + 1, Slices.allocate(8), 0);
703 }
704
705 @Test(expectedExceptions = InvalidCodePointException.class, expectedExceptionsMessageRegExp = "Invalid code point 0xFFFFFFBF")
706 public void testSetCodePointContinuationByte()
707 {
708 setCodePointAt(CONTINUATION_BYTE, Slices.allocate(8), 0);
709 }
710
711 }