Debian Codegraph @

src/backend/utils/adt/pg_locale_icu.c

raw
1 /*-----------------------------------------------------------------------
2 *
3 * PostgreSQL locale utilities for ICU
4 *
5 * Portions Copyright (c) 2002-2025, PostgreSQL Global Development Group
6 *
7 * src/backend/utils/adt/pg_locale_icu.c
8 *
9 *-----------------------------------------------------------------------
10 */
11
12 #include "postgres.h"
13
14 #ifdef USE_ICU
15 #include <unicode/ucnv.h>
16 #include <unicode/ustring.h>
17
18 /*
19 * ucol_strcollUTF8() was introduced in ICU 50, but it is buggy before ICU 53.
20 * (see
21 * <https://www.postgresql.org/message-id/flat/f1438ec6-22aa-4029-9a3b-26f79d330e72%40manitou-mail.org>)
22 */
23 #if U_ICU_VERSION_MAJOR_NUM >= 53
24 #define HAVE_UCOL_STRCOLLUTF8 1
25 #else
26 #undef HAVE_UCOL_STRCOLLUTF8
27 #endif
28
29 #endif
30
31 #include "access/htup_details.h"
32 #include "catalog/pg_database.h"
33 #include "catalog/pg_collation.h"
34 #include "mb/pg_wchar.h"
35 #include "miscadmin.h"
36 #include "utils/builtins.h"
37 #include "utils/formatting.h"
38 #include "utils/memutils.h"
39 #include "utils/pg_locale.h"
40 #include "utils/syscache.h"
41
42 /*
43 * Size of stack buffer to use for string transformations, used to avoid heap
44 * allocations in typical cases. This should be large enough that most strings
45 * will fit, but small enough that we feel comfortable putting it on the
46 * stack.
47 */
48 #define TEXTBUFLEN 1024
49
50 extern pg_locale_t create_pg_locale_icu(Oid collid, MemoryContext context);
51 extern size_t strlower_icu(char *dest, size_t destsize, const char *src,
52 ssize_t srclen, pg_locale_t locale);
53 extern size_t strtitle_icu(char *dest, size_t destsize, const char *src,
54 ssize_t srclen, pg_locale_t locale);
55 extern size_t strupper_icu(char *dest, size_t destsize, const char *src,
56 ssize_t srclen, pg_locale_t locale);
57 extern size_t strfold_icu(char *dest, size_t destsize, const char *src,
58 ssize_t srclen, pg_locale_t locale);
59
60 #ifdef USE_ICU
61
62 extern UCollator *pg_ucol_open(const char *loc_str);
63
64 static size_t strnxfrm_icu(char *dest, size_t destsize,
65 const char *src, ssize_t srclen,
66 pg_locale_t locale);
67 extern char *get_collation_actual_version_icu(const char *collcollate);
68
69 typedef int32_t (*ICU_Convert_Func) (UChar *dest, int32_t destCapacity,
70 const UChar *src, int32_t srcLength,
71 const char *locale,
72 UErrorCode *pErrorCode);
73
74 /*
75 * Converter object for converting between ICU's UChar strings and C strings
76 * in database encoding. Since the database encoding doesn't change, we only
77 * need one of these per session.
78 */
79 static UConverter *icu_converter = NULL;
80
81 static UCollator *make_icu_collator(const char *iculocstr,
82 const char *icurules);
83 static int strncoll_icu(const char *arg1, ssize_t len1,
84 const char *arg2, ssize_t len2,
85 pg_locale_t locale);
86 static size_t strnxfrm_prefix_icu(char *dest, size_t destsize,
87 const char *src, ssize_t srclen,
88 pg_locale_t locale);
89 #ifdef HAVE_UCOL_STRCOLLUTF8
90 static int strncoll_icu_utf8(const char *arg1, ssize_t len1,
91 const char *arg2, ssize_t len2,
92 pg_locale_t locale);
93 #endif
94 static size_t strnxfrm_prefix_icu_utf8(char *dest, size_t destsize,
95 const char *src, ssize_t srclen,
96 pg_locale_t locale);
97 static void init_icu_converter(void);
98 static int32_t uchar_length(UConverter *converter,
99 const char *str, int32_t len);
100 static int32_t uchar_convert(UConverter *converter,
101 UChar *dest, int32_t destlen,
102 const char *src, int32_t srclen);
103 static int32_t icu_to_uchar(UChar **buff_uchar, const char *buff,
104 size_t nbytes);
105 static size_t icu_from_uchar(char *dest, size_t destsize,
106 const UChar *buff_uchar, int32_t len_uchar);
107 static void icu_set_collation_attributes(UCollator *collator, const char *loc,
108 UErrorCode *status);
109 static int32_t icu_convert_case(ICU_Convert_Func func, pg_locale_t mylocale,
110 UChar **buff_dest, UChar *buff_source,
111 int32_t len_source);
112 static int32_t u_strToTitle_default_BI(UChar *dest, int32_t destCapacity,
113 const UChar *src, int32_t srcLength,
114 const char *locale,
115 UErrorCode *pErrorCode);
116 static int32_t u_strFoldCase_default(UChar *dest, int32_t destCapacity,
117 const UChar *src, int32_t srcLength,
118 const char *locale,
119 UErrorCode *pErrorCode);
120
121 static const struct collate_methods collate_methods_icu = {
122 .strncoll = strncoll_icu,
123 .strnxfrm = strnxfrm_icu,
124 .strnxfrm_prefix = strnxfrm_prefix_icu,
125 .strxfrm_is_safe = true,
126 };
127
128 static const struct collate_methods collate_methods_icu_utf8 = {
129 #ifdef HAVE_UCOL_STRCOLLUTF8
130 .strncoll = strncoll_icu_utf8,
131 #else
132 .strncoll = strncoll_icu,
133 #endif
134 .strnxfrm = strnxfrm_icu,
135 .strnxfrm_prefix = strnxfrm_prefix_icu_utf8,
136 .strxfrm_is_safe = true,
137 };
138
139 #endif
140
141 pg_locale_t
142 create_pg_locale_icu(Oid collid, MemoryContext context)
143 {
144 #ifdef USE_ICU
145 bool deterministic;
146 const char *iculocstr;
147 const char *icurules = NULL;
148 UCollator *collator;
149 pg_locale_t result;
150
151 if (collid == DEFAULT_COLLATION_OID)
152 {
153 HeapTuple tp;
154 Datum datum;
155 bool isnull;
156
157 tp = SearchSysCache1(DATABASEOID, ObjectIdGetDatum(MyDatabaseId));
158 if (!HeapTupleIsValid(tp))
159 elog(ERROR, "cache lookup failed for database %u", MyDatabaseId);
160
161 /* default database collation is always deterministic */
162 deterministic = true;
163 datum = SysCacheGetAttrNotNull(DATABASEOID, tp,
164 Anum_pg_database_datlocale);
165 iculocstr = TextDatumGetCString(datum);
166 datum = SysCacheGetAttr(DATABASEOID, tp,
167 Anum_pg_database_daticurules, &isnull);
168 if (!isnull)
169 icurules = TextDatumGetCString(datum);
170
171 ReleaseSysCache(tp);
172 }
173 else
174 {
175 Form_pg_collation collform;
176 HeapTuple tp;
177 Datum datum;
178 bool isnull;
179
180 tp = SearchSysCache1(COLLOID, ObjectIdGetDatum(collid));
181 if (!HeapTupleIsValid(tp))
182 elog(ERROR, "cache lookup failed for collation %u", collid);
183 collform = (Form_pg_collation) GETSTRUCT(tp);
184 deterministic = collform->collisdeterministic;
185 datum = SysCacheGetAttrNotNull(COLLOID, tp,
186 Anum_pg_collation_colllocale);
187 iculocstr = TextDatumGetCString(datum);
188 datum = SysCacheGetAttr(COLLOID, tp,
189 Anum_pg_collation_collicurules, &isnull);
190 if (!isnull)
191 icurules = TextDatumGetCString(datum);
192
193 ReleaseSysCache(tp);
194 }
195
196 collator = make_icu_collator(iculocstr, icurules);
197
198 result = MemoryContextAllocZero(context, sizeof(struct pg_locale_struct));
199 result->info.icu.locale = MemoryContextStrdup(context, iculocstr);
200 result->info.icu.ucol = collator;
201 result->provider = COLLPROVIDER_ICU;
202 result->deterministic = deterministic;
203 result->collate_is_c = false;
204 result->ctype_is_c = false;
205 if (GetDatabaseEncoding() == PG_UTF8)
206 result->collate = &collate_methods_icu_utf8;
207 else
208 result->collate = &collate_methods_icu;
209
210 return result;
211 #else
212 /* could get here if a collation was created by a build with ICU */
213 ereport(ERROR,
214 (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
215 errmsg("ICU is not supported in this build")));
216
217 return NULL;
218 #endif
219 }
220
221 #ifdef USE_ICU
222
223 /*
224 * Wrapper around ucol_open() to handle API differences for older ICU
225 * versions.
226 *
227 * Ensure that no path leaks a UCollator.
228 */
229 UCollator *
230 pg_ucol_open(const char *loc_str)
231 {
232 UCollator *collator;
233 UErrorCode status;
234 const char *orig_str = loc_str;
235 char *fixed_str = NULL;
236
237 /*
238 * Must never open default collator, because it depends on the environment
239 * and may change at any time. Should not happen, but check here to catch
240 * bugs that might be hard to catch otherwise.
241 *
242 * NB: the default collator is not the same as the collator for the root
243 * locale. The root locale may be specified as the empty string, "und", or
244 * "root". The default collator is opened by passing NULL to ucol_open().
245 */
246 if (loc_str == NULL)
247 elog(ERROR, "opening default collator is not supported");
248
249 /*
250 * In ICU versions 54 and earlier, "und" is not a recognized spelling of
251 * the root locale. If the first component of the locale is "und", replace
252 * with "root" before opening.
253 */
254 if (U_ICU_VERSION_MAJOR_NUM < 55)
255 {
256 char lang[ULOC_LANG_CAPACITY];
257
258 status = U_ZERO_ERROR;
259 uloc_getLanguage(loc_str, lang, ULOC_LANG_CAPACITY, &status);
260 if (U_FAILURE(status) || status == U_STRING_NOT_TERMINATED_WARNING)
261 {
262 ereport(ERROR,
263 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
264 errmsg("could not get language from locale \"%s\": %s",
265 loc_str, u_errorName(status))));
266 }
267
268 if (strcmp(lang, "und") == 0)
269 {
270 const char *remainder = loc_str + strlen("und");
271
272 fixed_str = palloc(strlen("root") + strlen(remainder) + 1);
273 strcpy(fixed_str, "root");
274 strcat(fixed_str, remainder);
275
276 loc_str = fixed_str;
277 }
278 }
279
280 status = U_ZERO_ERROR;
281 collator = ucol_open(loc_str, &status);
282 if (U_FAILURE(status))
283 ereport(ERROR,
284 /* use original string for error report */
285 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
286 errmsg("could not open collator for locale \"%s\": %s",
287 orig_str, u_errorName(status))));
288
289 if (U_ICU_VERSION_MAJOR_NUM < 54)
290 {
291 status = U_ZERO_ERROR;
292 icu_set_collation_attributes(collator, loc_str, &status);
293
294 /*
295 * Pretend the error came from ucol_open(), for consistent error
296 * message across ICU versions.
297 */
298 if (U_FAILURE(status) || status == U_STRING_NOT_TERMINATED_WARNING)
299 {
300 ucol_close(collator);
301 ereport(ERROR,
302 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
303 errmsg("could not open collator for locale \"%s\": %s",
304 orig_str, u_errorName(status))));
305 }
306 }
307
308 if (fixed_str != NULL)
309 pfree(fixed_str);
310
311 return collator;
312 }
313
314 /*
315 * Create a UCollator with the given locale string and rules.
316 *
317 * Ensure that no path leaks a UCollator.
318 */
319 static UCollator *
320 make_icu_collator(const char *iculocstr, const char *icurules)
321 {
322 if (!icurules)
323 {
324 /* simple case without rules */
325 return pg_ucol_open(iculocstr);
326 }
327 else
328 {
329 UCollator *collator_std_rules;
330 UCollator *collator_all_rules;
331 const UChar *std_rules;
332 UChar *my_rules;
333 UChar *all_rules;
334 int32_t length;
335 int32_t total;
336 UErrorCode status;
337
338 /*
339 * If rules are specified, we extract the rules of the standard
340 * collation, add our own rules, and make a new collator with the
341 * combined rules.
342 */
343 icu_to_uchar(&my_rules, icurules, strlen(icurules));
344
345 collator_std_rules = pg_ucol_open(iculocstr);
346
347 std_rules = ucol_getRules(collator_std_rules, &length);
348
349 total = u_strlen(std_rules) + u_strlen(my_rules) + 1;
350
351 /* avoid leaking collator on OOM */
352 all_rules = palloc_array_extended(UChar, total, MCXT_ALLOC_NO_OOM);
353 if (!all_rules)
354 {
355 ucol_close(collator_std_rules);
356 ereport(ERROR,
357 (errcode(ERRCODE_OUT_OF_MEMORY),
358 errmsg("out of memory")));
359 }
360
361 u_strcpy(all_rules, std_rules);
362 u_strcat(all_rules, my_rules);
363
364 ucol_close(collator_std_rules);
365
366 status = U_ZERO_ERROR;
367 collator_all_rules = ucol_openRules(all_rules, u_strlen(all_rules),
368 UCOL_DEFAULT, UCOL_DEFAULT_STRENGTH,
369 NULL, &status);
370 if (U_FAILURE(status))
371 {
372 ereport(ERROR,
373 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
374 errmsg("could not open collator for locale \"%s\" with rules \"%s\": %s",
375 iculocstr, icurules, u_errorName(status))));
376 }
377
378 pfree(my_rules);
379 pfree(all_rules);
380 return collator_all_rules;
381 }
382 }
383
384 size_t
385 strlower_icu(char *dest, size_t destsize, const char *src, ssize_t srclen,
386 pg_locale_t locale)
387 {
388 int32_t len_uchar;
389 int32_t len_conv;
390 UChar *buff_uchar;
391 UChar *buff_conv;
392 size_t result_len;
393
394 len_uchar = icu_to_uchar(&buff_uchar, src, srclen);
395 len_conv = icu_convert_case(u_strToLower, locale,
396 &buff_conv, buff_uchar, len_uchar);
397 result_len = icu_from_uchar(dest, destsize, buff_conv, len_conv);
398 pfree(buff_uchar);
399 pfree(buff_conv);
400
401 return result_len;
402 }
403
404 size_t
405 strtitle_icu(char *dest, size_t destsize, const char *src, ssize_t srclen,
406 pg_locale_t locale)
407 {
408 int32_t len_uchar;
409 int32_t len_conv;
410 UChar *buff_uchar;
411 UChar *buff_conv;
412 size_t result_len;
413
414 len_uchar = icu_to_uchar(&buff_uchar, src, srclen);
415 len_conv = icu_convert_case(u_strToTitle_default_BI, locale,
416 &buff_conv, buff_uchar, len_uchar);
417 result_len = icu_from_uchar(dest, destsize, buff_conv, len_conv);
418 pfree(buff_uchar);
419 pfree(buff_conv);
420
421 return result_len;
422 }
423
424 size_t
425 strupper_icu(char *dest, size_t destsize, const char *src, ssize_t srclen,
426 pg_locale_t locale)
427 {
428 int32_t len_uchar;
429 int32_t len_conv;
430 UChar *buff_uchar;
431 UChar *buff_conv;
432 size_t result_len;
433
434 len_uchar = icu_to_uchar(&buff_uchar, src, srclen);
435 len_conv = icu_convert_case(u_strToUpper, locale,
436 &buff_conv, buff_uchar, len_uchar);
437 result_len = icu_from_uchar(dest, destsize, buff_conv, len_conv);
438 pfree(buff_uchar);
439 pfree(buff_conv);
440
441 return result_len;
442 }
443
444 size_t
445 strfold_icu(char *dest, size_t destsize, const char *src, ssize_t srclen,
446 pg_locale_t locale)
447 {
448 int32_t len_uchar;
449 int32_t len_conv;
450 UChar *buff_uchar;
451 UChar *buff_conv;
452 size_t result_len;
453
454 len_uchar = icu_to_uchar(&buff_uchar, src, srclen);
455 len_conv = icu_convert_case(u_strFoldCase_default, locale,
456 &buff_conv, buff_uchar, len_uchar);
457 result_len = icu_from_uchar(dest, destsize, buff_conv, len_conv);
458 pfree(buff_uchar);
459 pfree(buff_conv);
460
461 return result_len;
462 }
463
464 /*
465 * strncoll_icu_utf8
466 *
467 * Call ucol_strcollUTF8() or ucol_strcoll() as appropriate for the given
468 * database encoding. An argument length of -1 means the string is
469 * NUL-terminated.
470 */
471 #ifdef HAVE_UCOL_STRCOLLUTF8
472 int
473 strncoll_icu_utf8(const char *arg1, ssize_t len1, const char *arg2, ssize_t len2,
474 pg_locale_t locale)
475 {
476 int result;
477 UErrorCode status;
478
479 Assert(locale->provider == COLLPROVIDER_ICU);
480
481 Assert(GetDatabaseEncoding() == PG_UTF8);
482
483 status = U_ZERO_ERROR;
484 result = ucol_strcollUTF8(locale->info.icu.ucol,
485 arg1, len1,
486 arg2, len2,
487 &status);
488 if (U_FAILURE(status))
489 ereport(ERROR,
490 (errmsg("collation failed: %s", u_errorName(status))));
491
492 return result;
493 }
494 #endif
495
496 /* 'srclen' of -1 means the strings are NUL-terminated */
497 size_t
498 strnxfrm_icu(char *dest, size_t destsize, const char *src, ssize_t srclen,
499 pg_locale_t locale)
500 {
501 UChar sbuf[TEXTBUFLEN / sizeof(UChar)];
502 UChar *uchar = sbuf;
503 int32_t ulen;
504 Size result_bsize;
505
506 Assert(locale->provider == COLLPROVIDER_ICU);
507
508 init_icu_converter();
509
510 ulen = uchar_length(icu_converter, src, srclen);
511
512 if (ulen >= lengthof(sbuf))
513 uchar = palloc_array(UChar, ulen + 1);
514
515 ulen = uchar_convert(icu_converter, uchar, ulen + 1, src, srclen);
516
517 result_bsize = ucol_getSortKey(locale->info.icu.ucol,
518 uchar, ulen,
519 (uint8_t *) dest, destsize);
520
521 /*
522 * ucol_getSortKey() counts the nul-terminator in the result length, but
523 * this function should not.
524 */
525 Assert(result_bsize > 0);
526 result_bsize--;
527
528 if (uchar != sbuf)
529 pfree(uchar);
530
531 /* if dest is defined, it should be nul-terminated */
532 Assert(result_bsize >= destsize || dest[result_bsize] == '\0');
533
534 return result_bsize;
535 }
536
537 /* 'srclen' of -1 means the strings are NUL-terminated */
538 size_t
539 strnxfrm_prefix_icu_utf8(char *dest, size_t destsize,
540 const char *src, ssize_t srclen,
541 pg_locale_t locale)
542 {
543 size_t result;
544 UCharIterator iter;
545 uint32_t state[2];
546 UErrorCode status;
547
548 Assert(locale->provider == COLLPROVIDER_ICU);
549
550 Assert(GetDatabaseEncoding() == PG_UTF8);
551
552 uiter_setUTF8(&iter, src, srclen);
553 state[0] = state[1] = 0; /* won't need that again */
554 status = U_ZERO_ERROR;
555 result = ucol_nextSortKeyPart(locale->info.icu.ucol,
556 &iter,
557 state,
558 (uint8_t *) dest,
559 destsize,
560 &status);
561 if (U_FAILURE(status))
562 ereport(ERROR,
563 (errmsg("sort key generation failed: %s",
564 u_errorName(status))));
565
566 return result;
567 }
568
569 char *
570 get_collation_actual_version_icu(const char *collcollate)
571 {
572 UCollator *collator;
573 UVersionInfo versioninfo;
574 char buf[U_MAX_VERSION_STRING_LENGTH];
575
576 collator = pg_ucol_open(collcollate);
577
578 ucol_getVersion(collator, versioninfo);
579 ucol_close(collator);
580
581 u_versionToString(versioninfo, buf);
582 return pstrdup(buf);
583 }
584
585 /*
586 * Convert a string in the database encoding into a string of UChars.
587 *
588 * The source string at buff is of length nbytes
589 * (it needn't be nul-terminated)
590 *
591 * *buff_uchar receives a pointer to the palloc'd result string, and
592 * the function's result is the number of UChars generated.
593 *
594 * The result string is nul-terminated, though most callers rely on the
595 * result length instead.
596 */
597 static int32_t
598 icu_to_uchar(UChar **buff_uchar, const char *buff, size_t nbytes)
599 {
600 int32_t len_uchar;
601
602 init_icu_converter();
603
604 len_uchar = uchar_length(icu_converter, buff, nbytes);
605
606 *buff_uchar = palloc_array(UChar, len_uchar + 1);
607 len_uchar = uchar_convert(icu_converter,
608 *buff_uchar, len_uchar + 1, buff, nbytes);
609
610 return len_uchar;
611 }
612
613 /*
614 * Convert a string of UChars into the database encoding.
615 *
616 * The source string at buff_uchar is of length len_uchar
617 * (it needn't be nul-terminated)
618 *
619 * *result receives a pointer to the palloc'd result string, and the
620 * function's result is the number of bytes generated (not counting nul).
621 *
622 * The result string is nul-terminated.
623 */
624 static size_t
625 icu_from_uchar(char *dest, size_t destsize, const UChar *buff_uchar, int32_t len_uchar)
626 {
627 UErrorCode status;
628 int32_t len_result;
629
630 init_icu_converter();
631
632 status = U_ZERO_ERROR;
633 len_result = ucnv_fromUChars(icu_converter, NULL, 0,
634 buff_uchar, len_uchar, &status);
635 if (U_FAILURE(status) && status != U_BUFFER_OVERFLOW_ERROR)
636 ereport(ERROR,
637 (errmsg("%s failed: %s", "ucnv_fromUChars",
638 u_errorName(status))));
639
640 if (len_result + 1 > destsize)
641 return len_result;
642
643 status = U_ZERO_ERROR;
644 len_result = ucnv_fromUChars(icu_converter, dest, len_result + 1,
645 buff_uchar, len_uchar, &status);
646 if (U_FAILURE(status) ||
647 status == U_STRING_NOT_TERMINATED_WARNING)
648 ereport(ERROR,
649 (errmsg("%s failed: %s", "ucnv_fromUChars",
650 u_errorName(status))));
651
652 return len_result;
653 }
654
655 static int32_t
656 icu_convert_case(ICU_Convert_Func func, pg_locale_t mylocale,
657 UChar **buff_dest, UChar *buff_source, int32_t len_source)
658 {
659 UErrorCode status;
660 int32_t len_dest;
661
662 len_dest = len_source; /* try first with same length */
663 *buff_dest = palloc_array(UChar, len_dest);
664 status = U_ZERO_ERROR;
665 len_dest = func(*buff_dest, len_dest, buff_source, len_source,
666 mylocale->info.icu.locale, &status);
667 if (status == U_BUFFER_OVERFLOW_ERROR)
668 {
669 /* try again with adjusted length */
670 pfree(*buff_dest);
671 *buff_dest = palloc_array(UChar, len_dest);
672 status = U_ZERO_ERROR;
673 len_dest = func(*buff_dest, len_dest, buff_source, len_source,
674 mylocale->info.icu.locale, &status);
675 }
676 if (U_FAILURE(status))
677 ereport(ERROR,
678 (errmsg("case conversion failed: %s", u_errorName(status))));
679 return len_dest;
680 }
681
682 static int32_t
683 u_strToTitle_default_BI(UChar *dest, int32_t destCapacity,
684 const UChar *src, int32_t srcLength,
685 const char *locale,
686 UErrorCode *pErrorCode)
687 {
688 return u_strToTitle(dest, destCapacity, src, srcLength,
689 NULL, locale, pErrorCode);
690 }
691
692 static int32_t
693 u_strFoldCase_default(UChar *dest, int32_t destCapacity,
694 const UChar *src, int32_t srcLength,
695 const char *locale,
696 UErrorCode *pErrorCode)
697 {
698 uint32 options = U_FOLD_CASE_DEFAULT;
699 char lang[ULOC_LANG_CAPACITY];
700 UErrorCode status;
701
702 /*
703 * Unlike the ICU APIs for lowercasing, titlecasing, and uppercasing, case
704 * folding does not accept a locale. Instead it just supports a single
705 * option relevant to Turkic languages 'az' and 'tr'; check for those
706 * languages to enable the option.
707 */
708 status = U_ZERO_ERROR;
709 uloc_getLanguage(locale, lang, ULOC_LANG_CAPACITY, &status);
710 if (U_SUCCESS(status) && status != U_STRING_NOT_TERMINATED_WARNING)
711 {
712 /*
713 * The option name is confusing, but it causes u_strFoldCase to use
714 * the 'T' mappings, which are ignored for U_FOLD_CASE_DEFAULT.
715 */
716 if (strcmp(lang, "tr") == 0 || strcmp(lang, "az") == 0)
717 options = U_FOLD_CASE_EXCLUDE_SPECIAL_I;
718 }
719
720 return u_strFoldCase(dest, destCapacity, src, srcLength,
721 options, pErrorCode);
722 }
723
724 /*
725 * strncoll_icu
726 *
727 * Convert the arguments from the database encoding to UChar strings, then
728 * call ucol_strcoll(). An argument length of -1 means that the string is
729 * NUL-terminated.
730 *
731 * When the database encoding is UTF-8, and ICU supports ucol_strcollUTF8(),
732 * caller should call that instead.
733 */
734 static int
735 strncoll_icu(const char *arg1, ssize_t len1,
736 const char *arg2, ssize_t len2, pg_locale_t locale)
737 {
738 UChar sbuf[TEXTBUFLEN / sizeof(UChar)];
739 UChar *buf = sbuf;
740 int32_t ulen1;
741 int32_t ulen2;
742 size_t bufsize;
743 UChar *uchar1,
744 *uchar2;
745 int result;
746
747 Assert(locale->provider == COLLPROVIDER_ICU);
748
749 /* if encoding is UTF8, use more efficient strncoll_icu_utf8 */
750 #ifdef HAVE_UCOL_STRCOLLUTF8
751 Assert(GetDatabaseEncoding() != PG_UTF8);
752 #endif
753
754 init_icu_converter();
755
756 ulen1 = uchar_length(icu_converter, arg1, len1);
757 ulen2 = uchar_length(icu_converter, arg2, len2);
758
759 /* ulen1+1 or ulen2+1 doesn't risk overflow, but summing them might */
760 bufsize = add_size(ulen1 + 1, ulen2 + 1);
761 if (bufsize > lengthof(sbuf))
762 buf = palloc_array(UChar, bufsize);
763
764 uchar1 = buf;
765 uchar2 = buf + ulen1 + 1;
766
767 ulen1 = uchar_convert(icu_converter, uchar1, ulen1 + 1, arg1, len1);
768 ulen2 = uchar_convert(icu_converter, uchar2, ulen2 + 1, arg2, len2);
769
770 result = ucol_strcoll(locale->info.icu.ucol,
771 uchar1, ulen1,
772 uchar2, ulen2);
773
774 if (buf != sbuf)
775 pfree(buf);
776
777 return result;
778 }
779
780 /* 'srclen' of -1 means the strings are NUL-terminated */
781 static size_t
782 strnxfrm_prefix_icu(char *dest, size_t destsize,
783 const char *src, ssize_t srclen,
784 pg_locale_t locale)
785 {
786 UChar sbuf[TEXTBUFLEN / sizeof(UChar)];
787 UChar *uchar = sbuf;
788 UCharIterator iter;
789 uint32_t state[2];
790 UErrorCode status;
791 int32_t ulen;
792 Size result_bsize;
793
794 Assert(locale->provider == COLLPROVIDER_ICU);
795
796 /* if encoding is UTF8, use more efficient strnxfrm_prefix_icu_utf8 */
797 Assert(GetDatabaseEncoding() != PG_UTF8);
798
799 init_icu_converter();
800
801 ulen = uchar_length(icu_converter, src, srclen);
802
803 if (ulen >= lengthof(sbuf))
804 uchar = palloc_array(UChar, ulen + 1);
805
806 ulen = uchar_convert(icu_converter, uchar, ulen + 1, src, srclen);
807
808 uiter_setString(&iter, uchar, ulen);
809 state[0] = state[1] = 0; /* won't need that again */
810 status = U_ZERO_ERROR;
811 result_bsize = ucol_nextSortKeyPart(locale->info.icu.ucol,
812 &iter,
813 state,
814 (uint8_t *) dest,
815 destsize,
816 &status);
817 if (U_FAILURE(status))
818 ereport(ERROR,
819 (errmsg("sort key generation failed: %s",
820 u_errorName(status))));
821
822 if (uchar != sbuf)
823 pfree(uchar);
824
825 return result_bsize;
826 }
827
828 static void
829 init_icu_converter(void)
830 {
831 const char *icu_encoding_name;
832 UErrorCode status;
833 UConverter *conv;
834
835 if (icu_converter)
836 return; /* already done */
837
838 icu_encoding_name = get_encoding_name_for_icu(GetDatabaseEncoding());
839 if (!icu_encoding_name)
840 ereport(ERROR,
841 (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
842 errmsg("encoding \"%s\" not supported by ICU",
843 pg_encoding_to_char(GetDatabaseEncoding()))));
844
845 status = U_ZERO_ERROR;
846 conv = ucnv_open(icu_encoding_name, &status);
847 if (U_FAILURE(status))
848 ereport(ERROR,
849 (errmsg("could not open ICU converter for encoding \"%s\": %s",
850 icu_encoding_name, u_errorName(status))));
851
852 icu_converter = conv;
853 }
854
855 /*
856 * Find length, in UChars, of given string if converted to UChar string.
857 *
858 * A length of -1 indicates that the input string is NUL-terminated.
859 *
860 * Note: given the assumption that the input string fits in MaxAllocSize,
861 * the result cannot overflow int32_t. But callers must be careful about
862 * multiplying the result by sizeof(UChar).
863 */
864 static int32_t
865 uchar_length(UConverter *converter, const char *str, int32_t len)
866 {
867 UErrorCode status = U_ZERO_ERROR;
868 int32_t ulen;
869
870 ulen = ucnv_toUChars(converter, NULL, 0, str, len, &status);
871 if (U_FAILURE(status) && status != U_BUFFER_OVERFLOW_ERROR)
872 ereport(ERROR,
873 (errmsg("%s failed: %s", "ucnv_toUChars", u_errorName(status))));
874 return ulen;
875 }
876
877 /*
878 * Convert the given source string into a UChar string, stored in dest, and
879 * return the length (in UChars).
880 *
881 * A srclen of -1 indicates that the input string is NUL-terminated.
882 */
883 static int32_t
884 uchar_convert(UConverter *converter, UChar *dest, int32_t destlen,
885 const char *src, int32_t srclen)
886 {
887 UErrorCode status = U_ZERO_ERROR;
888 int32_t ulen;
889
890 ulen = ucnv_toUChars(converter, dest, destlen, src, srclen, &status);
891 if (U_FAILURE(status))
892 ereport(ERROR,
893 (errmsg("%s failed: %s", "ucnv_toUChars", u_errorName(status))));
894 return ulen;
895 }
896
897 /*
898 * Parse collation attributes from the given locale string and apply them to
899 * the open collator.
900 *
901 * First, the locale string is canonicalized to an ICU format locale ID such
902 * as "und@colStrength=primary;colCaseLevel=yes". Then, it parses and applies
903 * the key-value arguments.
904 *
905 * Starting with ICU version 54, the attributes are processed automatically by
906 * ucol_open(), so this is only necessary for emulating this behavior on older
907 * versions.
908 */
909 pg_attribute_unused()
910 static void
911 icu_set_collation_attributes(UCollator *collator, const char *loc,
912 UErrorCode *status)
913 {
914 int32_t len;
915 char *icu_locale_id;
916 char *lower_str;
917 char *str;
918 char *token;
919
920 /*
921 * The input locale may be a BCP 47 language tag, e.g.
922 * "und-u-kc-ks-level1", which expresses the same attributes in a
923 * different form. It will be converted to the equivalent ICU format
924 * locale ID, e.g. "und@colcaselevel=yes;colstrength=primary", by
925 * uloc_canonicalize().
926 */
927 *status = U_ZERO_ERROR;
928 len = uloc_canonicalize(loc, NULL, 0, status);
929 icu_locale_id = palloc(len + 1);
930 *status = U_ZERO_ERROR;
931 len = uloc_canonicalize(loc, icu_locale_id, len + 1, status);
932 if (U_FAILURE(*status) || *status == U_STRING_NOT_TERMINATED_WARNING)
933 return;
934
935 lower_str = asc_tolower(icu_locale_id, strlen(icu_locale_id));
936
937 pfree(icu_locale_id);
938
939 str = strchr(lower_str, '@');
940 if (!str)
941 return;
942 str++;
943
944 while ((token = strsep(&str, ";")))
945 {
946 char *e = strchr(token, '=');
947
948 if (e)
949 {
950 char *name;
951 char *value;
952 UColAttribute uattr;
953 UColAttributeValue uvalue;
954
955 *status = U_ZERO_ERROR;
956
957 *e = '\0';
958 name = token;
959 value = e + 1;
960
961 /*
962 * See attribute name and value lists in ICU i18n/coll.cpp
963 */
964 if (strcmp(name, "colstrength") == 0)
965 uattr = UCOL_STRENGTH;
966 else if (strcmp(name, "colbackwards") == 0)
967 uattr = UCOL_FRENCH_COLLATION;
968 else if (strcmp(name, "colcaselevel") == 0)
969 uattr = UCOL_CASE_LEVEL;
970 else if (strcmp(name, "colcasefirst") == 0)
971 uattr = UCOL_CASE_FIRST;
972 else if (strcmp(name, "colalternate") == 0)
973 uattr = UCOL_ALTERNATE_HANDLING;
974 else if (strcmp(name, "colnormalization") == 0)
975 uattr = UCOL_NORMALIZATION_MODE;
976 else if (strcmp(name, "colnumeric") == 0)
977 uattr = UCOL_NUMERIC_COLLATION;
978 else
979 /* ignore if unknown */
980 continue;
981
982 if (strcmp(value, "primary") == 0)
983 uvalue = UCOL_PRIMARY;
984 else if (strcmp(value, "secondary") == 0)
985 uvalue = UCOL_SECONDARY;
986 else if (strcmp(value, "tertiary") == 0)
987 uvalue = UCOL_TERTIARY;
988 else if (strcmp(value, "quaternary") == 0)
989 uvalue = UCOL_QUATERNARY;
990 else if (strcmp(value, "identical") == 0)
991 uvalue = UCOL_IDENTICAL;
992 else if (strcmp(value, "no") == 0)
993 uvalue = UCOL_OFF;
994 else if (strcmp(value, "yes") == 0)
995 uvalue = UCOL_ON;
996 else if (strcmp(value, "shifted") == 0)
997 uvalue = UCOL_SHIFTED;
998 else if (strcmp(value, "non-ignorable") == 0)
999 uvalue = UCOL_NON_IGNORABLE;
1000 else if (strcmp(value, "lower") == 0)
1001 uvalue = UCOL_LOWER_FIRST;
1002 else if (strcmp(value, "upper") == 0)
1003 uvalue = UCOL_UPPER_FIRST;
1004 else
1005 {
1006 *status = U_ILLEGAL_ARGUMENT_ERROR;
1007 break;
1008 }
1009
1010 ucol_setAttribute(collator, uattr, uvalue, status);
1011 }
1012 }
1013
1014 pfree(lower_str);
1015 }
1016
1017 #endif /* USE_ICU */