Debian Codegraph @

tests/testutf8.c

raw
1 /* Copyright (C) 2026 Daniel Markstedt <[email protected]>
2 * UTF-8 unit tests for the Better String Library
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions are met:
6 *
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 *
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * 3. Neither the name of bstrlib nor the names of its contributors may be
15 * used to endorse or promote products derived from this software
16 * without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
22 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28 * POSSIBILITY OF SUCH DAMAGE.
29 *
30 * Alternatively, the contents of this file may be used under the terms of
31 * GNU General Public License Version 2 (the "GPL").
32 */
33
34 /*
35 * This file is the C unit test for the UTF-8 modules (utf8util, buniutil).
36 *
37 * Test data quick reference:
38 * U+0041 'A' = 0x41 (1-byte ASCII)
39 * U+00A9 '©' = 0xC2 0xA9 (2-byte)
40 * U+20AC '€' = 0xE2 0x82 0xAC (3-byte)
41 * U+1F600 '😀' = 0xF0 0x9F 0x98 0x80 (4-byte)
42 * UTF-16 U+1F600 = { 0xD83D, 0xDE00 } (surrogate pair)
43 */
44
45 #ifdef HAVE_CONFIG_H
46 #include <config.h>
47 #endif
48
49 #include "buniutil.h"
50 #include "bstrlib.h"
51 #include "utf8util.h"
52 #include <check.h>
53 #include <stdlib.h>
54 #include <string.h>
55
56 /* -----------------------------------------------------------------------
57 * core_000: utf8IteratorInit — valid inputs and error inputs
58 * ----------------------------------------------------------------------- */
59 START_TEST(core_000)
60 {
61 struct utf8Iterator iter;
62 unsigned char data[] = "Hello";
63
64 /* NULL iter pointer must not crash */
65 utf8IteratorInit(NULL, data, 5);
66
67 /* Valid initialisation */
68 utf8IteratorInit(&iter, data, 5);
69 ck_assert_int_eq(iter.slen, 5);
70 ck_assert_int_eq(iter.next, 0);
71 ck_assert_int_eq(iter.start, -1);
72 ck_assert_int_eq(iter.error, 0);
73 ck_assert(iter.data == data);
74
75 /* NULL data → sentinel values */
76 utf8IteratorInit(&iter, NULL, 5);
77 ck_assert_int_eq(iter.slen, -1);
78 ck_assert_int_eq(iter.next, -1);
79 ck_assert_int_eq(iter.error, 1);
80
81 /* Negative slen → sentinel values */
82 utf8IteratorInit(&iter, data, -1);
83 ck_assert_int_eq(iter.slen, -1);
84 ck_assert_int_eq(iter.next, -1);
85 ck_assert_int_eq(iter.error, 1);
86
87 /* Zero-length string is valid */
88 utf8IteratorInit(&iter, data, 0);
89 ck_assert_int_eq(iter.slen, 0);
90 ck_assert_int_eq(iter.next, 0);
91 ck_assert_int_eq(iter.error, 0);
92 }
93 END_TEST
94
95 /* -----------------------------------------------------------------------
96 * core_001: utf8IteratorUninit — clears all fields; handles NULL gracefully
97 * ----------------------------------------------------------------------- */
98 START_TEST(core_001)
99 {
100 struct utf8Iterator iter;
101 unsigned char data[] = "Hello";
102
103 utf8IteratorInit(&iter, data, 5);
104 utf8IteratorUninit(&iter);
105 ck_assert(iter.data == NULL);
106 ck_assert_int_eq(iter.slen, -1);
107 ck_assert_int_eq(iter.start, -1);
108 ck_assert_int_eq(iter.next, -1);
109
110 /* NULL pointer must not crash */
111 utf8IteratorUninit(NULL);
112 }
113 END_TEST
114
115 /* -----------------------------------------------------------------------
116 * core_002: utf8IteratorGetNextCodePoint — ASCII string iteration
117 * ----------------------------------------------------------------------- */
118 START_TEST(core_002)
119 {
120 struct utf8Iterator iter;
121 unsigned char data[] = "ABC";
122 cpUcs4 cp;
123
124 utf8IteratorInit(&iter, data, 3);
125
126 cp = utf8IteratorGetNextCodePoint(&iter, '?');
127 ck_assert_int_eq(cp, 'A');
128 ck_assert_int_eq(iter.error, 0);
129 ck_assert_int_eq(iter.start, 0);
130 ck_assert_int_eq(iter.next, 1);
131
132 cp = utf8IteratorGetNextCodePoint(&iter, '?');
133 ck_assert_int_eq(cp, 'B');
134 ck_assert_int_eq(iter.start, 1);
135 ck_assert_int_eq(iter.next, 2);
136
137 cp = utf8IteratorGetNextCodePoint(&iter, '?');
138 ck_assert_int_eq(cp, 'C');
139 ck_assert_int_eq(iter.start, 2);
140 ck_assert_int_eq(iter.next, 3);
141
142 /* Past end → errCh */
143 cp = utf8IteratorGetNextCodePoint(&iter, '?');
144 ck_assert_int_eq(cp, '?');
145
146 /* NULL iterator → errCh */
147 cp = utf8IteratorGetNextCodePoint(NULL, '?');
148 ck_assert_int_eq(cp, '?');
149 }
150 END_TEST
151
152 /* -----------------------------------------------------------------------
153 * core_003: utf8IteratorGetNextCodePoint — multi-byte sequences
154 *
155 * Sequence: © (U+00A9, 2-byte) € (U+20AC, 3-byte) 😀 (U+1F600, 4-byte)
156 * Bytes: C2 A9 E2 82 AC F0 9F 98 80
157 * ----------------------------------------------------------------------- */
158 START_TEST(core_003)
159 {
160 struct utf8Iterator iter;
161 /* © € 😀 */
162 unsigned char data[] = {
163 0xC2, 0xA9, /* U+00A9 © */
164 0xE2, 0x82, 0xAC, /* U+20AC € */
165 0xF0, 0x9F, 0x98, 0x80 /* U+1F600 😀 */
166 };
167 cpUcs4 cp;
168
169 utf8IteratorInit(&iter, data, sizeof(data));
170
171 cp = utf8IteratorGetNextCodePoint(&iter, '?');
172 ck_assert_int_eq(cp, 0x00A9);
173 ck_assert_int_eq(iter.error, 0);
174 ck_assert_int_eq(iter.start, 0);
175 ck_assert_int_eq(iter.next, 2);
176
177 cp = utf8IteratorGetNextCodePoint(&iter, '?');
178 ck_assert_int_eq(cp, 0x20AC);
179 ck_assert_int_eq(iter.error, 0);
180 ck_assert_int_eq(iter.start, 2);
181 ck_assert_int_eq(iter.next, 5);
182
183 cp = utf8IteratorGetNextCodePoint(&iter, '?');
184 ck_assert_int_eq(cp, 0x1F600);
185 ck_assert_int_eq(iter.error, 0);
186 ck_assert_int_eq(iter.start, 5);
187 ck_assert_int_eq(iter.next, 9);
188
189 /* Exhausted */
190 cp = utf8IteratorGetNextCodePoint(&iter, '?');
191 ck_assert_int_eq(cp, '?');
192 }
193 END_TEST
194
195 /* -----------------------------------------------------------------------
196 * core_004: utf8IteratorGetNextCodePoint — invalid byte sequences
197 *
198 * 0x80 alone is a stray continuation byte (invalid lead).
199 * 0xFF is never valid in UTF-8.
200 * ----------------------------------------------------------------------- */
201 START_TEST(core_004)
202 {
203 struct utf8Iterator iter;
204 /* stray continuation, then a valid ASCII char */
205 unsigned char data_cont[] = { 0x80, 0x41 };
206 /* 0xFF is always invalid */
207 unsigned char data_ff[] = { 0xFF, 0x41 };
208 cpUcs4 cp;
209
210 /* Stray continuation byte → error, iterator skips to next valid lead */
211 utf8IteratorInit(&iter, data_cont, sizeof(data_cont));
212 cp = utf8IteratorGetNextCodePoint(&iter, '?');
213 ck_assert_int_eq(cp, '?');
214 ck_assert_int_eq(iter.error, 1);
215 /* After error the iterator should have advanced past the bad byte(s) */
216 ck_assert(iter.next > 0);
217
218 /* 0xFF lead byte → error */
219 utf8IteratorInit(&iter, data_ff, sizeof(data_ff));
220 cp = utf8IteratorGetNextCodePoint(&iter, '?');
221 ck_assert_int_eq(cp, '?');
222 ck_assert_int_eq(iter.error, 1);
223 }
224 END_TEST
225
226 /* -----------------------------------------------------------------------
227 * core_005: utf8IteratorGetNextCodePoint — truncated sequence bounds checks
228 *
229 * The backing arrays contain full valid code points, but slen is set so the
230 * sequence is truncated at the end. Iterator must treat each as invalid and
231 * return errCh instead of decoding bytes past slen.
232 * ----------------------------------------------------------------------- */
233 START_TEST(core_005)
234 {
235 struct utf8Iterator iter;
236 cpUcs4 cp;
237
238 {
239 unsigned char data[] = { 0xC2, 0xA9 };
240 utf8IteratorInit(&iter, data, 1);
241 cp = utf8IteratorGetNextCodePoint(&iter, '?');
242 ck_assert_int_eq(cp, '?');
243 ck_assert_int_eq(iter.error, 1);
244 ck_assert_int_eq(iter.start, 0);
245 ck_assert_int_eq(iter.next, 1);
246 }
247
248 {
249 unsigned char data[] = { 0xE2, 0x82, 0xAC };
250 utf8IteratorInit(&iter, data, 2);
251 cp = utf8IteratorGetNextCodePoint(&iter, '?');
252 ck_assert_int_eq(cp, '?');
253 ck_assert_int_eq(iter.error, 1);
254 ck_assert_int_eq(iter.start, 0);
255 ck_assert_int_eq(iter.next, 2);
256 }
257
258 {
259 unsigned char data[] = { 0xF0, 0x9F, 0x98, 0x80 };
260 utf8IteratorInit(&iter, data, 3);
261 cp = utf8IteratorGetNextCodePoint(&iter, '?');
262 ck_assert_int_eq(cp, '?');
263 ck_assert_int_eq(iter.error, 1);
264 ck_assert_int_eq(iter.start, 0);
265 ck_assert_int_eq(iter.next, 3);
266 }
267 }
268 END_TEST
269
270 /* -----------------------------------------------------------------------
271 * core_006: utf8IteratorGetCurrCodePoint — truncated sequence bounds checks
272 *
273 * Peek must never decode bytes beyond slen. For each truncated sequence, it
274 * should return errCh, set iter.error, and leave iter.next unchanged.
275 * ----------------------------------------------------------------------- */
276 START_TEST(core_006)
277 {
278 struct utf8Iterator iter;
279 cpUcs4 cp;
280
281 {
282 unsigned char data[] = { 0xC2, 0xA9 };
283 utf8IteratorInit(&iter, data, 1);
284 cp = utf8IteratorGetCurrCodePoint(&iter, '?');
285 ck_assert_int_eq(cp, '?');
286 ck_assert_int_eq(iter.error, 1);
287 ck_assert_int_eq(iter.next, 0);
288 }
289
290 {
291 unsigned char data[] = { 0xE2, 0x82, 0xAC };
292 utf8IteratorInit(&iter, data, 2);
293 cp = utf8IteratorGetCurrCodePoint(&iter, '?');
294 ck_assert_int_eq(cp, '?');
295 ck_assert_int_eq(iter.error, 1);
296 ck_assert_int_eq(iter.next, 0);
297 }
298
299 {
300 unsigned char data[] = { 0xF0, 0x9F, 0x98, 0x80 };
301 utf8IteratorInit(&iter, data, 3);
302 cp = utf8IteratorGetCurrCodePoint(&iter, '?');
303 ck_assert_int_eq(cp, '?');
304 ck_assert_int_eq(iter.error, 1);
305 ck_assert_int_eq(iter.next, 0);
306 }
307 }
308 END_TEST
309
310 /* -----------------------------------------------------------------------
311 * core_007: utf8IteratorGetCurrCodePoint — peek without advancing
312 * ----------------------------------------------------------------------- */
313 START_TEST(core_007)
314 {
315 struct utf8Iterator iter;
316 unsigned char data[] = { 0xC2, 0xA9, 0x41 }; /* © A */
317 cpUcs4 cp;
318
319 utf8IteratorInit(&iter, data, sizeof(data));
320
321 /* Peek twice at the same position — must not advance */
322 cp = utf8IteratorGetCurrCodePoint(&iter, '?');
323 ck_assert_int_eq(cp, 0x00A9);
324 ck_assert_int_eq(iter.next, 0); /* still at start */
325
326 cp = utf8IteratorGetCurrCodePoint(&iter, '?');
327 ck_assert_int_eq(cp, 0x00A9);
328 ck_assert_int_eq(iter.next, 0);
329
330 /* Now advance with GetNext, then peek the next char */
331 utf8IteratorGetNextCodePoint(&iter, '?');
332 ck_assert_int_eq(iter.next, 2);
333
334 cp = utf8IteratorGetCurrCodePoint(&iter, '?');
335 ck_assert_int_eq(cp, 0x41); /* 'A' */
336 ck_assert_int_eq(iter.next, 2); /* still not advanced */
337
338 /* NULL iterator → errCh */
339 cp = utf8IteratorGetCurrCodePoint(NULL, '?');
340 ck_assert_int_eq(cp, '?');
341 }
342 END_TEST
343
344 /* -----------------------------------------------------------------------
345 * core_008: utf8ScanBackwardsForCodePoint — various positions
346 *
347 * Data: © (0xC2 0xA9) at bytes 0-1, then 'A' (0x41) at byte 2
348 * ----------------------------------------------------------------------- */
349 START_TEST(core_008)
350 {
351 unsigned char data[] = { 0xC2, 0xA9, 0x41 }; /* © A */
352 cpUcs4 out;
353 int ret;
354
355 /* pos=0 is the lead byte of © — ret=0, out=0xA9 */
356 ret = utf8ScanBackwardsForCodePoint(data, 3, 0, &out);
357 ck_assert_int_eq(ret, 0);
358 ck_assert_int_eq(out, 0x00A9);
359
360 /* pos=1 is the continuation byte — ret=1 (1 byte back to lead), out=0xA9 */
361 ret = utf8ScanBackwardsForCodePoint(data, 3, 1, &out);
362 ck_assert_int_eq(ret, 1);
363 ck_assert_int_eq(out, 0x00A9);
364
365 /* pos=2 is ASCII 'A' — ret=0, out='A' */
366 ret = utf8ScanBackwardsForCodePoint(data, 3, 2, &out);
367 ck_assert_int_eq(ret, 0);
368 ck_assert_int_eq(out, 0x41);
369
370 /* NULL msg → error (negative) */
371 ret = utf8ScanBackwardsForCodePoint(NULL, 3, 0, &out);
372 ck_assert(ret < 0);
373
374 /* pos out of range → error */
375 ret = utf8ScanBackwardsForCodePoint(data, 3, 3, &out);
376 ck_assert(ret < 0);
377
378 /* out=NULL is accepted; return value indicates success/failure */
379 ret = utf8ScanBackwardsForCodePoint(data, 3, 2, NULL);
380 ck_assert_int_eq(ret, 0);
381 }
382 END_TEST
383
384 /* -----------------------------------------------------------------------
385 * core_009: buIsUTF8Content
386 * ----------------------------------------------------------------------- */
387 START_TEST(core_009)
388 {
389 bstring b;
390 int ret;
391
392 /* NULL bstring → 0 */
393 ret = buIsUTF8Content(NULL);
394 ck_assert_int_eq(ret, 0);
395
396 /* Empty string → 1 (vacuously true) */
397 b = bfromcstr("");
398 ck_assert(b != NULL);
399 ret = buIsUTF8Content(b);
400 ck_assert_int_eq(ret, 1);
401 bdestroy(b);
402
403 /* Pure ASCII → 1 */
404 b = bfromcstr("Hello, world!");
405 ck_assert(b != NULL);
406 ret = buIsUTF8Content(b);
407 ck_assert_int_eq(ret, 1);
408 bdestroy(b);
409
410 /* Valid multi-byte UTF-8: © € 😀 */
411 {
412 unsigned char utf8[] = {
413 0xC2, 0xA9,
414 0xE2, 0x82, 0xAC,
415 0xF0, 0x9F, 0x98, 0x80
416 };
417 b = blk2bstr(utf8, sizeof(utf8));
418 ck_assert(b != NULL);
419 ret = buIsUTF8Content(b);
420 ck_assert_int_eq(ret, 1);
421 bdestroy(b);
422 }
423
424 /* Invalid: stray 0x80 continuation byte → 0 */
425 {
426 unsigned char bad[] = { 0x41, 0x80, 0x41 };
427 b = blk2bstr(bad, sizeof(bad));
428 ck_assert(b != NULL);
429 ret = buIsUTF8Content(b);
430 ck_assert_int_eq(ret, 0);
431 bdestroy(b);
432 }
433
434 /* Invalid: truncated 2-byte sequence → 0 */
435 {
436 unsigned char bad[] = { 0xC2 }; /* lead without continuation */
437 b = blk2bstr(bad, sizeof(bad));
438 ck_assert(b != NULL);
439 ret = buIsUTF8Content(b);
440 ck_assert_int_eq(ret, 0);
441 bdestroy(b);
442 }
443 }
444 END_TEST
445
446 /* -----------------------------------------------------------------------
447 * core_010: buAppendBlkUcs4 — UCS-4 array → UTF-8 bstring
448 * ----------------------------------------------------------------------- */
449 START_TEST(core_010)
450 {
451 bstring b;
452 int ret;
453
454 /* NULL arguments → BSTR_ERR */
455 b = bfromcstr("");
456 ck_assert(b != NULL);
457 ret = buAppendBlkUcs4(NULL, NULL, 0, '?');
458 ck_assert_int_eq(ret, BSTR_ERR);
459 ret = buAppendBlkUcs4(b, NULL, 1, '?');
460 ck_assert_int_eq(ret, BSTR_ERR);
461 bdestroy(b);
462
463 /* ASCII code points */
464 {
465 cpUcs4 pts[] = { 'H', 'i' };
466 b = bfromcstr("");
467 ck_assert(b != NULL);
468 ret = buAppendBlkUcs4(b, pts, 2, '?');
469 ck_assert_int_eq(ret, BSTR_OK);
470 ck_assert_int_eq(b->slen, 2);
471 ck_assert_int_eq(b->data[0], 'H');
472 ck_assert_int_eq(b->data[1], 'i');
473 bdestroy(b);
474 }
475
476 /* Mixed: © (U+00A9) and € (U+20AC) */
477 {
478 cpUcs4 pts[] = { 0x00A9, 0x20AC };
479 unsigned char expected[] = {
480 0xC2, 0xA9, /* © */
481 0xE2, 0x82, 0xAC /* € */
482 };
483 b = bfromcstr("");
484 ck_assert(b != NULL);
485 ret = buAppendBlkUcs4(b, pts, 2, '?');
486 ck_assert_int_eq(ret, BSTR_OK);
487 ck_assert_int_eq(b->slen, 5);
488 ret = memcmp(b->data, expected, 5);
489 ck_assert_int_eq(ret, 0);
490 bdestroy(b);
491 }
492
493 /* 4-byte: 😀 (U+1F600) */
494 {
495 cpUcs4 pts[] = { 0x1F600 };
496 unsigned char expected[] = { 0xF0, 0x9F, 0x98, 0x80 };
497 b = bfromcstr("");
498 ck_assert(b != NULL);
499 ret = buAppendBlkUcs4(b, pts, 1, '?');
500 ck_assert_int_eq(ret, BSTR_OK);
501 ck_assert_int_eq(b->slen, 4);
502 ret = memcmp(b->data, expected, 4);
503 ck_assert_int_eq(ret, 0);
504 bdestroy(b);
505 }
506
507 /* Invalid code point with valid errCh → substituted */
508 {
509 cpUcs4 pts[] = { 0xD800 }; /* surrogates are illegal */
510 b = bfromcstr("");
511 ck_assert(b != NULL);
512 ret = buAppendBlkUcs4(b, pts, 1, '?');
513 ck_assert_int_eq(ret, BSTR_OK);
514 ck_assert_int_eq(b->slen, 1);
515 ck_assert_int_eq(b->data[0], '?');
516 bdestroy(b);
517 }
518
519 /* Invalid code point with invalid errCh → BSTR_ERR, bstring unchanged */
520 {
521 cpUcs4 pts[] = { 0xD800 };
522 b = bfromcstr("pre");
523 ck_assert(b != NULL);
524 ret = buAppendBlkUcs4(b, pts, 1, 0xD800); /* errCh also invalid */
525 ck_assert_int_eq(ret, BSTR_ERR);
526 /* slen must be rolled back */
527 ck_assert_int_eq(b->slen, 3);
528 bdestroy(b);
529 }
530
531 /* Zero-length array → BSTR_OK, nothing appended */
532 {
533 cpUcs4 pts[] = { 'X' };
534 b = bfromcstr("pre");
535 ck_assert(b != NULL);
536 ret = buAppendBlkUcs4(b, pts, 0, '?');
537 ck_assert_int_eq(ret, BSTR_OK);
538 ck_assert_int_eq(b->slen, 3);
539 bdestroy(b);
540 }
541 }
542 END_TEST
543
544 /* -----------------------------------------------------------------------
545 * core_011: buGetBlkUTF16 — UTF-8 bstring → UTF-16 array
546 * ----------------------------------------------------------------------- */
547 START_TEST(core_011)
548 {
549 cpUcs2 buf[16];
550 int ret;
551
552 /* NULL arguments → BSTR_ERR */
553 {
554 unsigned char raw[] = { 0x41 };
555 bstring b = blk2bstr(raw, 1);
556 ck_assert(b != NULL);
557 ret = buGetBlkUTF16(NULL, 4, '?', b, 0);
558 ck_assert_int_eq(ret, BSTR_ERR);
559 ret = buGetBlkUTF16(buf, 0, '?', b, 0);
560 ck_assert_int_eq(ret, BSTR_ERR);
561 ret = buGetBlkUTF16(buf, 4, '?', NULL, 0);
562 ck_assert_int_eq(ret, BSTR_ERR);
563 ret = buGetBlkUTF16(buf, 4, '?', b, -1);
564 ck_assert_int_eq(ret, BSTR_ERR);
565 bdestroy(b);
566 }
567
568 /* ASCII "AB" → UTF-16 { 0x0041, 0x0042, 0, ... } */
569 {
570 bstring b = bfromcstr("AB");
571 ck_assert(b != NULL);
572 memset(buf, 0xFF, sizeof(buf));
573 ret = buGetBlkUTF16(buf, 4, '?', b, 0);
574 ck_assert_int_eq(ret, 2);
575 ck_assert_int_eq(buf[0], 0x0041);
576 ck_assert_int_eq(buf[1], 0x0042);
577 ck_assert_int_eq(buf[2], 0); /* null-padded */
578 bdestroy(b);
579 }
580
581 /* © € → UTF-16 BMP values (U+00A9, U+20AC) */
582 {
583 unsigned char raw[] = {
584 0xC2, 0xA9,
585 0xE2, 0x82, 0xAC
586 };
587 bstring b = blk2bstr(raw, sizeof(raw));
588 ck_assert(b != NULL);
589 memset(buf, 0xFF, sizeof(buf));
590 ret = buGetBlkUTF16(buf, 4, '?', b, 0);
591 ck_assert_int_eq(ret, 2);
592 ck_assert_int_eq(buf[0], 0x00A9);
593 ck_assert_int_eq(buf[1], 0x20AC);
594 bdestroy(b);
595 }
596
597 /* pos=1 skips first code point */
598 {
599 bstring b = bfromcstr("AB");
600 ck_assert(b != NULL);
601 memset(buf, 0, sizeof(buf));
602 ret = buGetBlkUTF16(buf, 4, '?', b, 1);
603 ck_assert_int_eq(ret, 1);
604 ck_assert_int_eq(buf[0], 0x0042);
605 bdestroy(b);
606 }
607
608 /* Supplementary character 😀 (U+1F600) → surrogate pair */
609 {
610 unsigned char raw[] = { 0xF0, 0x9F, 0x98, 0x80 };
611 bstring b = blk2bstr(raw, sizeof(raw));
612 ck_assert(b != NULL);
613 memset(buf, 0, sizeof(buf));
614 ret = buGetBlkUTF16(buf, 4, '?', b, 0);
615 ck_assert_int_eq(ret, 2); /* one code point → two UTF-16 units */
616 ck_assert_int_eq(buf[0], 0xD83D); /* high surrogate */
617 ck_assert_int_eq(buf[1], 0xDE00); /* low surrogate */
618 bdestroy(b);
619 }
620 }
621 END_TEST
622
623 /* -----------------------------------------------------------------------
624 * core_012: buAppendBlkUTF16 — UTF-16 array → UTF-8 bstring
625 * ----------------------------------------------------------------------- */
626 START_TEST(core_012)
627 {
628 bstring b;
629 int ret;
630
631 /* NULL / bad arguments → BSTR_ERR */
632 b = bfromcstr("");
633 ck_assert(b != NULL);
634 {
635 cpUcs2 u[] = { 0x0041 };
636 ret = buAppendBlkUTF16(NULL, u, 1, NULL, '?');
637 ck_assert_int_eq(ret, BSTR_ERR);
638 ret = buAppendBlkUTF16(b, NULL, 1, NULL, '?');
639 ck_assert_int_eq(ret, BSTR_ERR);
640 ret = buAppendBlkUTF16(b, u, -1, NULL, '?');
641 ck_assert_int_eq(ret, BSTR_ERR);
642 }
643 bdestroy(b);
644
645 /* Zero-length input → BSTR_OK, nothing appended */
646 {
647 cpUcs2 u[] = { 0x0041 };
648 b = bfromcstr("pre");
649 ck_assert(b != NULL);
650 ret = buAppendBlkUTF16(b, u, 0, NULL, '?');
651 ck_assert_int_eq(ret, BSTR_OK);
652 ck_assert_int_eq(b->slen, 3);
653 bdestroy(b);
654 }
655
656 /* ASCII "AB" in UTF-16 → "AB" in UTF-8 */
657 {
658 cpUcs2 u[] = { 0x0041, 0x0042 };
659 b = bfromcstr("");
660 ck_assert(b != NULL);
661 ret = buAppendBlkUTF16(b, u, 2, NULL, '?');
662 ck_assert_int_eq(ret, BSTR_OK);
663 ck_assert_int_eq(b->slen, 2);
664 ck_assert_int_eq(b->data[0], 'A');
665 ck_assert_int_eq(b->data[1], 'B');
666 bdestroy(b);
667 }
668
669 /* BMP characters: U+00A9 © and U+20AC € */
670 {
671 cpUcs2 u[] = { 0x00A9, 0x20AC };
672 unsigned char expected[] = {
673 0xC2, 0xA9,
674 0xE2, 0x82, 0xAC
675 };
676 b = bfromcstr("");
677 ck_assert(b != NULL);
678 ret = buAppendBlkUTF16(b, u, 2, NULL, '?');
679 ck_assert_int_eq(ret, BSTR_OK);
680 ck_assert_int_eq(b->slen, 5);
681 ret = memcmp(b->data, expected, 5);
682 ck_assert_int_eq(ret, 0);
683 bdestroy(b);
684 }
685
686 /* Surrogate pair: 😀 (U+1F600) = { 0xD83D, 0xDE00 } */
687 {
688 cpUcs2 u[] = { 0xD83D, 0xDE00 };
689 unsigned char expected[] = { 0xF0, 0x9F, 0x98, 0x80 };
690 b = bfromcstr("");
691 ck_assert(b != NULL);
692 ret = buAppendBlkUTF16(b, u, 2, NULL, '?');
693 ck_assert_int_eq(ret, BSTR_OK);
694 ck_assert_int_eq(b->slen, 4);
695 ret = memcmp(b->data, expected, 4);
696 ck_assert_int_eq(ret, 0);
697 bdestroy(b);
698 }
699
700 /* Little-endian BOM (0xFFFE) → byte-swapped input */
701 {
702 /* 'A' (0x0041) with bytes swapped = 0x4100, plus LE BOM */
703 cpUcs2 u[] = { 0xFFFE, 0x4100 };
704 b = bfromcstr("");
705 ck_assert(b != NULL);
706 ret = buAppendBlkUTF16(b, u, 2, NULL, '?');
707 ck_assert_int_eq(ret, BSTR_OK);
708 ck_assert_int_eq(b->slen, 1);
709 ck_assert_int_eq(b->data[0], 'A');
710 bdestroy(b);
711 }
712
713 /* Big-endian BOM (0xFEFF) is consumed and removed from output */
714 {
715 cpUcs2 u[] = { 0xFEFF, 0x0041 };
716 b = bfromcstr("");
717 ck_assert(b != NULL);
718 ret = buAppendBlkUTF16(b, u, 2, NULL, '?');
719 ck_assert_int_eq(ret, BSTR_OK);
720 ck_assert_int_eq(b->slen, 1);
721 ck_assert_int_eq(b->data[0], 'A');
722 bdestroy(b);
723 }
724
725 /* Invalid low surrogate alone with valid errCh → substituted */
726 {
727 cpUcs2 u[] = { 0xDC00 };
728 b = bfromcstr("");
729 ck_assert(b != NULL);
730 ret = buAppendBlkUTF16(b, u, 1, NULL, '?');
731 ck_assert_int_eq(ret, BSTR_OK);
732 ck_assert_int_eq(b->slen, 1);
733 ck_assert_int_eq(b->data[0], '?');
734 bdestroy(b);
735 }
736
737 /* Invalid low surrogate then ASCII with valid errCh */
738 {
739 cpUcs2 u[] = { 0xDC00, 0x0041 };
740 unsigned char expected[] = { '?', 'A' };
741 b = bfromcstr("");
742 ck_assert(b != NULL);
743 ret = buAppendBlkUTF16(b, u, 2, NULL, '?');
744 ck_assert_int_eq(ret, BSTR_OK);
745 ck_assert_int_eq(b->slen, 2);
746 ret = memcmp(b->data, expected, 2);
747 ck_assert_int_eq(ret, 0);
748 bdestroy(b);
749 }
750
751 /* Invalid surrogate with invalid errCh → BSTR_ERR and rollback */
752 {
753 cpUcs2 u[] = { 0xDC00 };
754 b = bfromcstr("pre");
755 ck_assert(b != NULL);
756 ret = buAppendBlkUTF16(b, u, 1, NULL, 0xD800); /* invalid errCh */
757 ck_assert_int_eq(ret, BSTR_ERR);
758 ck_assert_int_eq(b->slen, 3); /* unchanged */
759 ck_assert_int_eq(b->data[0], 'p');
760 ck_assert_int_eq(b->data[1], 'r');
761 ck_assert_int_eq(b->data[2], 'e');
762 bdestroy(b);
763 }
764
765 /* bom out-parameter gets set when BOM appears in stream */
766 {
767 cpUcs2 in_bom = 0;
768 cpUcs2 u[] = { 0xFEFF, 0x0041 };
769 b = bfromcstr("");
770 ck_assert(b != NULL);
771 ret = buAppendBlkUTF16(b, u, 2, &in_bom, '?');
772 ck_assert_int_eq(ret, BSTR_OK);
773 ck_assert_int_eq(in_bom, 0xFEFF);
774 ck_assert_int_eq(b->slen, 1);
775 ck_assert_int_eq(b->data[0], 'A');
776 bdestroy(b);
777 }
778
779 /* Pre-seeded bom controls endianness even without BOM in input */
780 {
781 cpUcs2 in_bom = 0xFFFE;
782 cpUcs2 u[] = { 0x4100 }; /* bytes for 0x0041 in opposite endian */
783 b = bfromcstr("");
784 ck_assert(b != NULL);
785 ret = buAppendBlkUTF16(b, u, 1, &in_bom, '?');
786 ck_assert_int_eq(ret, BSTR_OK);
787 ck_assert_int_eq(in_bom, 0xFFFE); /* preserved */
788 ck_assert_int_eq(b->slen, 1);
789 ck_assert_int_eq(b->data[0], 'A');
790 bdestroy(b);
791 }
792
793 /* Larger than TEMP_UCS4_BUFFER_SIZE exercises internal flush path */
794 {
795 cpUcs2 u[80];
796 for (int j = 0; j < 80; j++) {
797 u[j] = (cpUcs2)('A' + (j % 26));
798 }
799 b = bfromcstr("");
800 ck_assert(b != NULL);
801 ret = buAppendBlkUTF16(b, u, 80, NULL, '?');
802 ck_assert_int_eq(ret, BSTR_OK);
803 ck_assert_int_eq(b->slen, 80);
804 for (int j = 0; j < 80; j++) {
805 ck_assert_int_eq(b->data[j], 'A' + (j % 26));
806 }
807 bdestroy(b);
808 }
809 }
810 END_TEST
811
812 /* -----------------------------------------------------------------------
813 * core_013: regression guard
814 *
815 * Guard against regressions for:
816 * high surrogate followed by non-low surrogate.
817 *
818 * Expected behavior is:
819 * first code unit substituted with errCh, second processed normally.
820 * ----------------------------------------------------------------------- */
821 START_TEST(core_013)
822 {
823 bstring b;
824 int ret;
825
826 {
827 cpUcs2 u[] = { 0xD83D, 0x0041 };
828 unsigned char expected[] = { '?', 'A' };
829 b = bfromcstr("");
830 ck_assert(b != NULL);
831 ret = buAppendBlkUTF16(b, u, 2, NULL, '?');
832 ck_assert_int_eq(ret, BSTR_OK);
833 ck_assert_int_eq(b->slen, 2);
834 ret = memcmp(b->data, expected, 2);
835 ck_assert_int_eq(ret, 0);
836 bdestroy(b);
837 }
838 }
839 END_TEST
840
841 /* -----------------------------------------------------------------------
842 * core_013: utf8ScanBackwardsForCodePoint — invalid continuation bytes
843 *
844 * Each case starts at a lead byte but includes one or more non-continuation
845 * trailing bytes. Scanner must reject these and return an error.
846 * ----------------------------------------------------------------------- */
847 START_TEST(core_014)
848 {
849 cpUcs4 out = 0;
850 int ret;
851
852 /* Invalid 2-byte sequence: second byte must be 10xxxxxx */
853 {
854 unsigned char data[] = { 0xC2, 0x41 };
855 ret = utf8ScanBackwardsForCodePoint(data, 2, 0, &out);
856 ck_assert(ret < 0);
857 }
858
859 /* Invalid 3-byte sequence: middle byte must be 10xxxxxx */
860 {
861 unsigned char data[] = { 0xE2, 0x28, 0xAC };
862 ret = utf8ScanBackwardsForCodePoint(data, 3, 0, &out);
863 ck_assert(ret < 0);
864 }
865
866 /* Invalid 4-byte sequence: third byte must be 10xxxxxx */
867 {
868 unsigned char data[] = { 0xF0, 0x9F, 0x41, 0x80 };
869 ret = utf8ScanBackwardsForCodePoint(data, 4, 0, &out);
870 ck_assert(ret < 0);
871 }
872 }
873 END_TEST
874
875 int
876 main(void)
877 {
878 /* Build test suite */
879 Suite *suite = suite_create("bstr-utf8");
880 /* Core tests */
881 TCase *core = tcase_create("Core");
882 tcase_add_test(core, core_000);
883 tcase_add_test(core, core_001);
884 tcase_add_test(core, core_002);
885 tcase_add_test(core, core_003);
886 tcase_add_test(core, core_004);
887 tcase_add_test(core, core_005);
888 tcase_add_test(core, core_006);
889 tcase_add_test(core, core_007);
890 tcase_add_test(core, core_008);
891 tcase_add_test(core, core_009);
892 tcase_add_test(core, core_010);
893 tcase_add_test(core, core_011);
894 tcase_add_test(core, core_012);
895 tcase_add_test(core, core_013);
896 tcase_add_test(core, core_014);
897 suite_add_tcase(suite, core);
898 /* Run tests */
899 SRunner *runner = srunner_create(suite);
900 srunner_run_all(runner, CK_ENV);
901 int number_failed = srunner_ntests_failed(runner);
902 srunner_free(runner);
903 return (0 == number_failed) ? EXIT_SUCCESS : EXIT_FAILURE;
904 }