| 1 |
/*----------------------------------------------------------------------- |
| 2 |
* |
| 3 |
* PostgreSQL locale utilities for ICU |
| 4 |
* |
| 5 |
* Portions Copyright (c) 2002-2025, PostgreSQL Global Development Group |
| 6 |
* |
| 7 |
* src/backend/utils/adt/pg_locale_icu.c |
| 8 |
* |
| 9 |
*----------------------------------------------------------------------- |
| 10 |
*/ |
| 11 |
|
| 12 |
#include "postgres.h" |
| 13 |
|
| 14 |
#ifdef USE_ICU |
| 15 |
#include <unicode/ucnv.h> |
| 16 |
#include <unicode/ustring.h> |
| 17 |
|
| 18 |
/* |
| 19 |
* ucol_strcollUTF8() was introduced in ICU 50, but it is buggy before ICU 53. |
| 20 |
* (see |
| 21 |
* <https://www.postgresql.org/message-id/flat/f1438ec6-22aa-4029-9a3b-26f79d330e72%40manitou-mail.org>) |
| 22 |
*/ |
| 23 |
#if U_ICU_VERSION_MAJOR_NUM >= 53 |
| 24 |
#define HAVE_UCOL_STRCOLLUTF8 1 |
| 25 |
#else |
| 26 |
#undef HAVE_UCOL_STRCOLLUTF8 |
| 27 |
#endif |
| 28 |
|
| 29 |
#endif |
| 30 |
|
| 31 |
#include "access/htup_details.h" |
| 32 |
#include "catalog/pg_database.h" |
| 33 |
#include "catalog/pg_collation.h" |
| 34 |
#include "mb/pg_wchar.h" |
| 35 |
#include "miscadmin.h" |
| 36 |
#include "utils/builtins.h" |
| 37 |
#include "utils/formatting.h" |
| 38 |
#include "utils/memutils.h" |
| 39 |
#include "utils/pg_locale.h" |
| 40 |
#include "utils/syscache.h" |
| 41 |
|
| 42 |
/* |
| 43 |
* Size of stack buffer to use for string transformations, used to avoid heap |
| 44 |
* allocations in typical cases. This should be large enough that most strings |
| 45 |
* will fit, but small enough that we feel comfortable putting it on the |
| 46 |
* stack. |
| 47 |
*/ |
| 48 |
#define TEXTBUFLEN 1024 |
| 49 |
|
| 50 |
extern pg_locale_t create_pg_locale_icu(Oid collid, MemoryContext context); |
| 51 |
extern size_t strlower_icu(char *dest, size_t destsize, const char *src, |
| 52 |
ssize_t srclen, pg_locale_t locale); |
| 53 |
extern size_t strtitle_icu(char *dest, size_t destsize, const char *src, |
| 54 |
ssize_t srclen, pg_locale_t locale); |
| 55 |
extern size_t strupper_icu(char *dest, size_t destsize, const char *src, |
| 56 |
ssize_t srclen, pg_locale_t locale); |
| 57 |
extern size_t strfold_icu(char *dest, size_t destsize, const char *src, |
| 58 |
ssize_t srclen, pg_locale_t locale); |
| 59 |
|
| 60 |
#ifdef USE_ICU |
| 61 |
|
| 62 |
extern UCollator *pg_ucol_open(const char *loc_str); |
| 63 |
|
| 64 |
static size_t strnxfrm_icu(char *dest, size_t destsize, |
| 65 |
const char *src, ssize_t srclen, |
| 66 |
pg_locale_t locale); |
| 67 |
extern char *get_collation_actual_version_icu(const char *collcollate); |
| 68 |
|
| 69 |
typedef int32_t (*ICU_Convert_Func) (UChar *dest, int32_t destCapacity, |
| 70 |
const UChar *src, int32_t srcLength, |
| 71 |
const char *locale, |
| 72 |
UErrorCode *pErrorCode); |
| 73 |
|
| 74 |
/* |
| 75 |
* Converter object for converting between ICU's UChar strings and C strings |
| 76 |
* in database encoding. Since the database encoding doesn't change, we only |
| 77 |
* need one of these per session. |
| 78 |
*/ |
| 79 |
static UConverter *icu_converter = NULL; |
| 80 |
|
| 81 |
static UCollator *make_icu_collator(const char *iculocstr, |
| 82 |
const char *icurules); |
| 83 |
static int strncoll_icu(const char *arg1, ssize_t len1, |
| 84 |
const char *arg2, ssize_t len2, |
| 85 |
pg_locale_t locale); |
| 86 |
static size_t strnxfrm_prefix_icu(char *dest, size_t destsize, |
| 87 |
const char *src, ssize_t srclen, |
| 88 |
pg_locale_t locale); |
| 89 |
#ifdef HAVE_UCOL_STRCOLLUTF8 |
| 90 |
static int strncoll_icu_utf8(const char *arg1, ssize_t len1, |
| 91 |
const char *arg2, ssize_t len2, |
| 92 |
pg_locale_t locale); |
| 93 |
#endif |
| 94 |
static size_t strnxfrm_prefix_icu_utf8(char *dest, size_t destsize, |
| 95 |
const char *src, ssize_t srclen, |
| 96 |
pg_locale_t locale); |
| 97 |
static void init_icu_converter(void); |
| 98 |
static int32_t uchar_length(UConverter *converter, |
| 99 |
const char *str, int32_t len); |
| 100 |
static int32_t uchar_convert(UConverter *converter, |
| 101 |
UChar *dest, int32_t destlen, |
| 102 |
const char *src, int32_t srclen); |
| 103 |
static int32_t icu_to_uchar(UChar **buff_uchar, const char *buff, |
| 104 |
size_t nbytes); |
| 105 |
static size_t icu_from_uchar(char *dest, size_t destsize, |
| 106 |
const UChar *buff_uchar, int32_t len_uchar); |
| 107 |
static void icu_set_collation_attributes(UCollator *collator, const char *loc, |
| 108 |
UErrorCode *status); |
| 109 |
static int32_t icu_convert_case(ICU_Convert_Func func, pg_locale_t mylocale, |
| 110 |
UChar **buff_dest, UChar *buff_source, |
| 111 |
int32_t len_source); |
| 112 |
static int32_t u_strToTitle_default_BI(UChar *dest, int32_t destCapacity, |
| 113 |
const UChar *src, int32_t srcLength, |
| 114 |
const char *locale, |
| 115 |
UErrorCode *pErrorCode); |
| 116 |
static int32_t u_strFoldCase_default(UChar *dest, int32_t destCapacity, |
| 117 |
const UChar *src, int32_t srcLength, |
| 118 |
const char *locale, |
| 119 |
UErrorCode *pErrorCode); |
| 120 |
|
| 121 |
static const struct collate_methods collate_methods_icu = { |
| 122 |
.strncoll = strncoll_icu, |
| 123 |
.strnxfrm = strnxfrm_icu, |
| 124 |
.strnxfrm_prefix = strnxfrm_prefix_icu, |
| 125 |
.strxfrm_is_safe = true, |
| 126 |
}; |
| 127 |
|
| 128 |
static const struct collate_methods collate_methods_icu_utf8 = { |
| 129 |
#ifdef HAVE_UCOL_STRCOLLUTF8 |
| 130 |
.strncoll = strncoll_icu_utf8, |
| 131 |
#else |
| 132 |
.strncoll = strncoll_icu, |
| 133 |
#endif |
| 134 |
.strnxfrm = strnxfrm_icu, |
| 135 |
.strnxfrm_prefix = strnxfrm_prefix_icu_utf8, |
| 136 |
.strxfrm_is_safe = true, |
| 137 |
}; |
| 138 |
|
| 139 |
#endif |
| 140 |
|
| 141 |
pg_locale_t |
| 142 |
create_pg_locale_icu(Oid collid, MemoryContext context) |
| 143 |
{ |
| 144 |
#ifdef USE_ICU |
| 145 |
bool deterministic; |
| 146 |
const char *iculocstr; |
| 147 |
const char *icurules = NULL; |
| 148 |
UCollator *collator; |
| 149 |
pg_locale_t result; |
| 150 |
|
| 151 |
if (collid == DEFAULT_COLLATION_OID) |
| 152 |
{ |
| 153 |
HeapTuple tp; |
| 154 |
Datum datum; |
| 155 |
bool isnull; |
| 156 |
|
| 157 |
tp = SearchSysCache1(DATABASEOID, ObjectIdGetDatum(MyDatabaseId)); |
| 158 |
if (!HeapTupleIsValid(tp)) |
| 159 |
elog(ERROR, "cache lookup failed for database %u", MyDatabaseId); |
| 160 |
|
| 161 |
/* default database collation is always deterministic */ |
| 162 |
deterministic = true; |
| 163 |
datum = SysCacheGetAttrNotNull(DATABASEOID, tp, |
| 164 |
Anum_pg_database_datlocale); |
| 165 |
iculocstr = TextDatumGetCString(datum); |
| 166 |
datum = SysCacheGetAttr(DATABASEOID, tp, |
| 167 |
Anum_pg_database_daticurules, &isnull); |
| 168 |
if (!isnull) |
| 169 |
icurules = TextDatumGetCString(datum); |
| 170 |
|
| 171 |
ReleaseSysCache(tp); |
| 172 |
} |
| 173 |
else |
| 174 |
{ |
| 175 |
Form_pg_collation collform; |
| 176 |
HeapTuple tp; |
| 177 |
Datum datum; |
| 178 |
bool isnull; |
| 179 |
|
| 180 |
tp = SearchSysCache1(COLLOID, ObjectIdGetDatum(collid)); |
| 181 |
if (!HeapTupleIsValid(tp)) |
| 182 |
elog(ERROR, "cache lookup failed for collation %u", collid); |
| 183 |
collform = (Form_pg_collation) GETSTRUCT(tp); |
| 184 |
deterministic = collform->collisdeterministic; |
| 185 |
datum = SysCacheGetAttrNotNull(COLLOID, tp, |
| 186 |
Anum_pg_collation_colllocale); |
| 187 |
iculocstr = TextDatumGetCString(datum); |
| 188 |
datum = SysCacheGetAttr(COLLOID, tp, |
| 189 |
Anum_pg_collation_collicurules, &isnull); |
| 190 |
if (!isnull) |
| 191 |
icurules = TextDatumGetCString(datum); |
| 192 |
|
| 193 |
ReleaseSysCache(tp); |
| 194 |
} |
| 195 |
|
| 196 |
collator = make_icu_collator(iculocstr, icurules); |
| 197 |
|
| 198 |
result = MemoryContextAllocZero(context, sizeof(struct pg_locale_struct)); |
| 199 |
result->info.icu.locale = MemoryContextStrdup(context, iculocstr); |
| 200 |
result->info.icu.ucol = collator; |
| 201 |
result->provider = COLLPROVIDER_ICU; |
| 202 |
result->deterministic = deterministic; |
| 203 |
result->collate_is_c = false; |
| 204 |
result->ctype_is_c = false; |
| 205 |
if (GetDatabaseEncoding() == PG_UTF8) |
| 206 |
result->collate = &collate_methods_icu_utf8; |
| 207 |
else |
| 208 |
result->collate = &collate_methods_icu; |
| 209 |
|
| 210 |
return result; |
| 211 |
#else |
| 212 |
/* could get here if a collation was created by a build with ICU */ |
| 213 |
ereport(ERROR, |
| 214 |
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED), |
| 215 |
errmsg("ICU is not supported in this build"))); |
| 216 |
|
| 217 |
return NULL; |
| 218 |
#endif |
| 219 |
} |
| 220 |
|
| 221 |
#ifdef USE_ICU |
| 222 |
|
| 223 |
/* |
| 224 |
* Wrapper around ucol_open() to handle API differences for older ICU |
| 225 |
* versions. |
| 226 |
* |
| 227 |
* Ensure that no path leaks a UCollator. |
| 228 |
*/ |
| 229 |
UCollator * |
| 230 |
pg_ucol_open(const char *loc_str) |
| 231 |
{ |
| 232 |
UCollator *collator; |
| 233 |
UErrorCode status; |
| 234 |
const char *orig_str = loc_str; |
| 235 |
char *fixed_str = NULL; |
| 236 |
|
| 237 |
/* |
| 238 |
* Must never open default collator, because it depends on the environment |
| 239 |
* and may change at any time. Should not happen, but check here to catch |
| 240 |
* bugs that might be hard to catch otherwise. |
| 241 |
* |
| 242 |
* NB: the default collator is not the same as the collator for the root |
| 243 |
* locale. The root locale may be specified as the empty string, "und", or |
| 244 |
* "root". The default collator is opened by passing NULL to ucol_open(). |
| 245 |
*/ |
| 246 |
if (loc_str == NULL) |
| 247 |
elog(ERROR, "opening default collator is not supported"); |
| 248 |
|
| 249 |
/* |
| 250 |
* In ICU versions 54 and earlier, "und" is not a recognized spelling of |
| 251 |
* the root locale. If the first component of the locale is "und", replace |
| 252 |
* with "root" before opening. |
| 253 |
*/ |
| 254 |
if (U_ICU_VERSION_MAJOR_NUM < 55) |
| 255 |
{ |
| 256 |
char lang[ULOC_LANG_CAPACITY]; |
| 257 |
|
| 258 |
status = U_ZERO_ERROR; |
| 259 |
uloc_getLanguage(loc_str, lang, ULOC_LANG_CAPACITY, &status); |
| 260 |
if (U_FAILURE(status) || status == U_STRING_NOT_TERMINATED_WARNING) |
| 261 |
{ |
| 262 |
ereport(ERROR, |
| 263 |
(errcode(ERRCODE_INVALID_PARAMETER_VALUE), |
| 264 |
errmsg("could not get language from locale \"%s\": %s", |
| 265 |
loc_str, u_errorName(status)))); |
| 266 |
} |
| 267 |
|
| 268 |
if (strcmp(lang, "und") == 0) |
| 269 |
{ |
| 270 |
const char *remainder = loc_str + strlen("und"); |
| 271 |
|
| 272 |
fixed_str = palloc(strlen("root") + strlen(remainder) + 1); |
| 273 |
strcpy(fixed_str, "root"); |
| 274 |
strcat(fixed_str, remainder); |
| 275 |
|
| 276 |
loc_str = fixed_str; |
| 277 |
} |
| 278 |
} |
| 279 |
|
| 280 |
status = U_ZERO_ERROR; |
| 281 |
collator = ucol_open(loc_str, &status); |
| 282 |
if (U_FAILURE(status)) |
| 283 |
ereport(ERROR, |
| 284 |
/* use original string for error report */ |
| 285 |
(errcode(ERRCODE_INVALID_PARAMETER_VALUE), |
| 286 |
errmsg("could not open collator for locale \"%s\": %s", |
| 287 |
orig_str, u_errorName(status)))); |
| 288 |
|
| 289 |
if (U_ICU_VERSION_MAJOR_NUM < 54) |
| 290 |
{ |
| 291 |
status = U_ZERO_ERROR; |
| 292 |
icu_set_collation_attributes(collator, loc_str, &status); |
| 293 |
|
| 294 |
/* |
| 295 |
* Pretend the error came from ucol_open(), for consistent error |
| 296 |
* message across ICU versions. |
| 297 |
*/ |
| 298 |
if (U_FAILURE(status) || status == U_STRING_NOT_TERMINATED_WARNING) |
| 299 |
{ |
| 300 |
ucol_close(collator); |
| 301 |
ereport(ERROR, |
| 302 |
(errcode(ERRCODE_INVALID_PARAMETER_VALUE), |
| 303 |
errmsg("could not open collator for locale \"%s\": %s", |
| 304 |
orig_str, u_errorName(status)))); |
| 305 |
} |
| 306 |
} |
| 307 |
|
| 308 |
if (fixed_str != NULL) |
| 309 |
pfree(fixed_str); |
| 310 |
|
| 311 |
return collator; |
| 312 |
} |
| 313 |
|
| 314 |
/* |
| 315 |
* Create a UCollator with the given locale string and rules. |
| 316 |
* |
| 317 |
* Ensure that no path leaks a UCollator. |
| 318 |
*/ |
| 319 |
static UCollator * |
| 320 |
make_icu_collator(const char *iculocstr, const char *icurules) |
| 321 |
{ |
| 322 |
if (!icurules) |
| 323 |
{ |
| 324 |
/* simple case without rules */ |
| 325 |
return pg_ucol_open(iculocstr); |
| 326 |
} |
| 327 |
else |
| 328 |
{ |
| 329 |
UCollator *collator_std_rules; |
| 330 |
UCollator *collator_all_rules; |
| 331 |
const UChar *std_rules; |
| 332 |
UChar *my_rules; |
| 333 |
UChar *all_rules; |
| 334 |
int32_t length; |
| 335 |
int32_t total; |
| 336 |
UErrorCode status; |
| 337 |
|
| 338 |
/* |
| 339 |
* If rules are specified, we extract the rules of the standard |
| 340 |
* collation, add our own rules, and make a new collator with the |
| 341 |
* combined rules. |
| 342 |
*/ |
| 343 |
icu_to_uchar(&my_rules, icurules, strlen(icurules)); |
| 344 |
|
| 345 |
collator_std_rules = pg_ucol_open(iculocstr); |
| 346 |
|
| 347 |
std_rules = ucol_getRules(collator_std_rules, &length); |
| 348 |
|
| 349 |
total = u_strlen(std_rules) + u_strlen(my_rules) + 1; |
| 350 |
|
| 351 |
/* avoid leaking collator on OOM */ |
| 352 |
all_rules = palloc_array_extended(UChar, total, MCXT_ALLOC_NO_OOM); |
| 353 |
if (!all_rules) |
| 354 |
{ |
| 355 |
ucol_close(collator_std_rules); |
| 356 |
ereport(ERROR, |
| 357 |
(errcode(ERRCODE_OUT_OF_MEMORY), |
| 358 |
errmsg("out of memory"))); |
| 359 |
} |
| 360 |
|
| 361 |
u_strcpy(all_rules, std_rules); |
| 362 |
u_strcat(all_rules, my_rules); |
| 363 |
|
| 364 |
ucol_close(collator_std_rules); |
| 365 |
|
| 366 |
status = U_ZERO_ERROR; |
| 367 |
collator_all_rules = ucol_openRules(all_rules, u_strlen(all_rules), |
| 368 |
UCOL_DEFAULT, UCOL_DEFAULT_STRENGTH, |
| 369 |
NULL, &status); |
| 370 |
if (U_FAILURE(status)) |
| 371 |
{ |
| 372 |
ereport(ERROR, |
| 373 |
(errcode(ERRCODE_INVALID_PARAMETER_VALUE), |
| 374 |
errmsg("could not open collator for locale \"%s\" with rules \"%s\": %s", |
| 375 |
iculocstr, icurules, u_errorName(status)))); |
| 376 |
} |
| 377 |
|
| 378 |
pfree(my_rules); |
| 379 |
pfree(all_rules); |
| 380 |
return collator_all_rules; |
| 381 |
} |
| 382 |
} |
| 383 |
|
| 384 |
size_t |
| 385 |
strlower_icu(char *dest, size_t destsize, const char *src, ssize_t srclen, |
| 386 |
pg_locale_t locale) |
| 387 |
{ |
| 388 |
int32_t len_uchar; |
| 389 |
int32_t len_conv; |
| 390 |
UChar *buff_uchar; |
| 391 |
UChar *buff_conv; |
| 392 |
size_t result_len; |
| 393 |
|
| 394 |
len_uchar = icu_to_uchar(&buff_uchar, src, srclen); |
| 395 |
len_conv = icu_convert_case(u_strToLower, locale, |
| 396 |
&buff_conv, buff_uchar, len_uchar); |
| 397 |
result_len = icu_from_uchar(dest, destsize, buff_conv, len_conv); |
| 398 |
pfree(buff_uchar); |
| 399 |
pfree(buff_conv); |
| 400 |
|
| 401 |
return result_len; |
| 402 |
} |
| 403 |
|
| 404 |
size_t |
| 405 |
strtitle_icu(char *dest, size_t destsize, const char *src, ssize_t srclen, |
| 406 |
pg_locale_t locale) |
| 407 |
{ |
| 408 |
int32_t len_uchar; |
| 409 |
int32_t len_conv; |
| 410 |
UChar *buff_uchar; |
| 411 |
UChar *buff_conv; |
| 412 |
size_t result_len; |
| 413 |
|
| 414 |
len_uchar = icu_to_uchar(&buff_uchar, src, srclen); |
| 415 |
len_conv = icu_convert_case(u_strToTitle_default_BI, locale, |
| 416 |
&buff_conv, buff_uchar, len_uchar); |
| 417 |
result_len = icu_from_uchar(dest, destsize, buff_conv, len_conv); |
| 418 |
pfree(buff_uchar); |
| 419 |
pfree(buff_conv); |
| 420 |
|
| 421 |
return result_len; |
| 422 |
} |
| 423 |
|
| 424 |
size_t |
| 425 |
strupper_icu(char *dest, size_t destsize, const char *src, ssize_t srclen, |
| 426 |
pg_locale_t locale) |
| 427 |
{ |
| 428 |
int32_t len_uchar; |
| 429 |
int32_t len_conv; |
| 430 |
UChar *buff_uchar; |
| 431 |
UChar *buff_conv; |
| 432 |
size_t result_len; |
| 433 |
|
| 434 |
len_uchar = icu_to_uchar(&buff_uchar, src, srclen); |
| 435 |
len_conv = icu_convert_case(u_strToUpper, locale, |
| 436 |
&buff_conv, buff_uchar, len_uchar); |
| 437 |
result_len = icu_from_uchar(dest, destsize, buff_conv, len_conv); |
| 438 |
pfree(buff_uchar); |
| 439 |
pfree(buff_conv); |
| 440 |
|
| 441 |
return result_len; |
| 442 |
} |
| 443 |
|
| 444 |
size_t |
| 445 |
strfold_icu(char *dest, size_t destsize, const char *src, ssize_t srclen, |
| 446 |
pg_locale_t locale) |
| 447 |
{ |
| 448 |
int32_t len_uchar; |
| 449 |
int32_t len_conv; |
| 450 |
UChar *buff_uchar; |
| 451 |
UChar *buff_conv; |
| 452 |
size_t result_len; |
| 453 |
|
| 454 |
len_uchar = icu_to_uchar(&buff_uchar, src, srclen); |
| 455 |
len_conv = icu_convert_case(u_strFoldCase_default, locale, |
| 456 |
&buff_conv, buff_uchar, len_uchar); |
| 457 |
result_len = icu_from_uchar(dest, destsize, buff_conv, len_conv); |
| 458 |
pfree(buff_uchar); |
| 459 |
pfree(buff_conv); |
| 460 |
|
| 461 |
return result_len; |
| 462 |
} |
| 463 |
|
| 464 |
/* |
| 465 |
* strncoll_icu_utf8 |
| 466 |
* |
| 467 |
* Call ucol_strcollUTF8() or ucol_strcoll() as appropriate for the given |
| 468 |
* database encoding. An argument length of -1 means the string is |
| 469 |
* NUL-terminated. |
| 470 |
*/ |
| 471 |
#ifdef HAVE_UCOL_STRCOLLUTF8 |
| 472 |
int |
| 473 |
strncoll_icu_utf8(const char *arg1, ssize_t len1, const char *arg2, ssize_t len2, |
| 474 |
pg_locale_t locale) |
| 475 |
{ |
| 476 |
int result; |
| 477 |
UErrorCode status; |
| 478 |
|
| 479 |
Assert(locale->provider == COLLPROVIDER_ICU); |
| 480 |
|
| 481 |
Assert(GetDatabaseEncoding() == PG_UTF8); |
| 482 |
|
| 483 |
status = U_ZERO_ERROR; |
| 484 |
result = ucol_strcollUTF8(locale->info.icu.ucol, |
| 485 |
arg1, len1, |
| 486 |
arg2, len2, |
| 487 |
&status); |
| 488 |
if (U_FAILURE(status)) |
| 489 |
ereport(ERROR, |
| 490 |
(errmsg("collation failed: %s", u_errorName(status)))); |
| 491 |
|
| 492 |
return result; |
| 493 |
} |
| 494 |
#endif |
| 495 |
|
| 496 |
/* 'srclen' of -1 means the strings are NUL-terminated */ |
| 497 |
size_t |
| 498 |
strnxfrm_icu(char *dest, size_t destsize, const char *src, ssize_t srclen, |
| 499 |
pg_locale_t locale) |
| 500 |
{ |
| 501 |
UChar sbuf[TEXTBUFLEN / sizeof(UChar)]; |
| 502 |
UChar *uchar = sbuf; |
| 503 |
int32_t ulen; |
| 504 |
Size result_bsize; |
| 505 |
|
| 506 |
Assert(locale->provider == COLLPROVIDER_ICU); |
| 507 |
|
| 508 |
init_icu_converter(); |
| 509 |
|
| 510 |
ulen = uchar_length(icu_converter, src, srclen); |
| 511 |
|
| 512 |
if (ulen >= lengthof(sbuf)) |
| 513 |
uchar = palloc_array(UChar, ulen + 1); |
| 514 |
|
| 515 |
ulen = uchar_convert(icu_converter, uchar, ulen + 1, src, srclen); |
| 516 |
|
| 517 |
result_bsize = ucol_getSortKey(locale->info.icu.ucol, |
| 518 |
uchar, ulen, |
| 519 |
(uint8_t *) dest, destsize); |
| 520 |
|
| 521 |
/* |
| 522 |
* ucol_getSortKey() counts the nul-terminator in the result length, but |
| 523 |
* this function should not. |
| 524 |
*/ |
| 525 |
Assert(result_bsize > 0); |
| 526 |
result_bsize--; |
| 527 |
|
| 528 |
if (uchar != sbuf) |
| 529 |
pfree(uchar); |
| 530 |
|
| 531 |
/* if dest is defined, it should be nul-terminated */ |
| 532 |
Assert(result_bsize >= destsize || dest[result_bsize] == '\0'); |
| 533 |
|
| 534 |
return result_bsize; |
| 535 |
} |
| 536 |
|
| 537 |
/* 'srclen' of -1 means the strings are NUL-terminated */ |
| 538 |
size_t |
| 539 |
strnxfrm_prefix_icu_utf8(char *dest, size_t destsize, |
| 540 |
const char *src, ssize_t srclen, |
| 541 |
pg_locale_t locale) |
| 542 |
{ |
| 543 |
size_t result; |
| 544 |
UCharIterator iter; |
| 545 |
uint32_t state[2]; |
| 546 |
UErrorCode status; |
| 547 |
|
| 548 |
Assert(locale->provider == COLLPROVIDER_ICU); |
| 549 |
|
| 550 |
Assert(GetDatabaseEncoding() == PG_UTF8); |
| 551 |
|
| 552 |
uiter_setUTF8(&iter, src, srclen); |
| 553 |
state[0] = state[1] = 0; /* won't need that again */ |
| 554 |
status = U_ZERO_ERROR; |
| 555 |
result = ucol_nextSortKeyPart(locale->info.icu.ucol, |
| 556 |
&iter, |
| 557 |
state, |
| 558 |
(uint8_t *) dest, |
| 559 |
destsize, |
| 560 |
&status); |
| 561 |
if (U_FAILURE(status)) |
| 562 |
ereport(ERROR, |
| 563 |
(errmsg("sort key generation failed: %s", |
| 564 |
u_errorName(status)))); |
| 565 |
|
| 566 |
return result; |
| 567 |
} |
| 568 |
|
| 569 |
char * |
| 570 |
get_collation_actual_version_icu(const char *collcollate) |
| 571 |
{ |
| 572 |
UCollator *collator; |
| 573 |
UVersionInfo versioninfo; |
| 574 |
char buf[U_MAX_VERSION_STRING_LENGTH]; |
| 575 |
|
| 576 |
collator = pg_ucol_open(collcollate); |
| 577 |
|
| 578 |
ucol_getVersion(collator, versioninfo); |
| 579 |
ucol_close(collator); |
| 580 |
|
| 581 |
u_versionToString(versioninfo, buf); |
| 582 |
return pstrdup(buf); |
| 583 |
} |
| 584 |
|
| 585 |
/* |
| 586 |
* Convert a string in the database encoding into a string of UChars. |
| 587 |
* |
| 588 |
* The source string at buff is of length nbytes |
| 589 |
* (it needn't be nul-terminated) |
| 590 |
* |
| 591 |
* *buff_uchar receives a pointer to the palloc'd result string, and |
| 592 |
* the function's result is the number of UChars generated. |
| 593 |
* |
| 594 |
* The result string is nul-terminated, though most callers rely on the |
| 595 |
* result length instead. |
| 596 |
*/ |
| 597 |
static int32_t |
| 598 |
icu_to_uchar(UChar **buff_uchar, const char *buff, size_t nbytes) |
| 599 |
{ |
| 600 |
int32_t len_uchar; |
| 601 |
|
| 602 |
init_icu_converter(); |
| 603 |
|
| 604 |
len_uchar = uchar_length(icu_converter, buff, nbytes); |
| 605 |
|
| 606 |
*buff_uchar = palloc_array(UChar, len_uchar + 1); |
| 607 |
len_uchar = uchar_convert(icu_converter, |
| 608 |
*buff_uchar, len_uchar + 1, buff, nbytes); |
| 609 |
|
| 610 |
return len_uchar; |
| 611 |
} |
| 612 |
|
| 613 |
/* |
| 614 |
* Convert a string of UChars into the database encoding. |
| 615 |
* |
| 616 |
* The source string at buff_uchar is of length len_uchar |
| 617 |
* (it needn't be nul-terminated) |
| 618 |
* |
| 619 |
* *result receives a pointer to the palloc'd result string, and the |
| 620 |
* function's result is the number of bytes generated (not counting nul). |
| 621 |
* |
| 622 |
* The result string is nul-terminated. |
| 623 |
*/ |
| 624 |
static size_t |
| 625 |
icu_from_uchar(char *dest, size_t destsize, const UChar *buff_uchar, int32_t len_uchar) |
| 626 |
{ |
| 627 |
UErrorCode status; |
| 628 |
int32_t len_result; |
| 629 |
|
| 630 |
init_icu_converter(); |
| 631 |
|
| 632 |
status = U_ZERO_ERROR; |
| 633 |
len_result = ucnv_fromUChars(icu_converter, NULL, 0, |
| 634 |
buff_uchar, len_uchar, &status); |
| 635 |
if (U_FAILURE(status) && status != U_BUFFER_OVERFLOW_ERROR) |
| 636 |
ereport(ERROR, |
| 637 |
(errmsg("%s failed: %s", "ucnv_fromUChars", |
| 638 |
u_errorName(status)))); |
| 639 |
|
| 640 |
if (len_result + 1 > destsize) |
| 641 |
return len_result; |
| 642 |
|
| 643 |
status = U_ZERO_ERROR; |
| 644 |
len_result = ucnv_fromUChars(icu_converter, dest, len_result + 1, |
| 645 |
buff_uchar, len_uchar, &status); |
| 646 |
if (U_FAILURE(status) || |
| 647 |
status == U_STRING_NOT_TERMINATED_WARNING) |
| 648 |
ereport(ERROR, |
| 649 |
(errmsg("%s failed: %s", "ucnv_fromUChars", |
| 650 |
u_errorName(status)))); |
| 651 |
|
| 652 |
return len_result; |
| 653 |
} |
| 654 |
|
| 655 |
static int32_t |
| 656 |
icu_convert_case(ICU_Convert_Func func, pg_locale_t mylocale, |
| 657 |
UChar **buff_dest, UChar *buff_source, int32_t len_source) |
| 658 |
{ |
| 659 |
UErrorCode status; |
| 660 |
int32_t len_dest; |
| 661 |
|
| 662 |
len_dest = len_source; /* try first with same length */ |
| 663 |
*buff_dest = palloc_array(UChar, len_dest); |
| 664 |
status = U_ZERO_ERROR; |
| 665 |
len_dest = func(*buff_dest, len_dest, buff_source, len_source, |
| 666 |
mylocale->info.icu.locale, &status); |
| 667 |
if (status == U_BUFFER_OVERFLOW_ERROR) |
| 668 |
{ |
| 669 |
/* try again with adjusted length */ |
| 670 |
pfree(*buff_dest); |
| 671 |
*buff_dest = palloc_array(UChar, len_dest); |
| 672 |
status = U_ZERO_ERROR; |
| 673 |
len_dest = func(*buff_dest, len_dest, buff_source, len_source, |
| 674 |
mylocale->info.icu.locale, &status); |
| 675 |
} |
| 676 |
if (U_FAILURE(status)) |
| 677 |
ereport(ERROR, |
| 678 |
(errmsg("case conversion failed: %s", u_errorName(status)))); |
| 679 |
return len_dest; |
| 680 |
} |
| 681 |
|
| 682 |
static int32_t |
| 683 |
u_strToTitle_default_BI(UChar *dest, int32_t destCapacity, |
| 684 |
const UChar *src, int32_t srcLength, |
| 685 |
const char *locale, |
| 686 |
UErrorCode *pErrorCode) |
| 687 |
{ |
| 688 |
return u_strToTitle(dest, destCapacity, src, srcLength, |
| 689 |
NULL, locale, pErrorCode); |
| 690 |
} |
| 691 |
|
| 692 |
static int32_t |
| 693 |
u_strFoldCase_default(UChar *dest, int32_t destCapacity, |
| 694 |
const UChar *src, int32_t srcLength, |
| 695 |
const char *locale, |
| 696 |
UErrorCode *pErrorCode) |
| 697 |
{ |
| 698 |
uint32 options = U_FOLD_CASE_DEFAULT; |
| 699 |
char lang[ULOC_LANG_CAPACITY]; |
| 700 |
UErrorCode status; |
| 701 |
|
| 702 |
/* |
| 703 |
* Unlike the ICU APIs for lowercasing, titlecasing, and uppercasing, case |
| 704 |
* folding does not accept a locale. Instead it just supports a single |
| 705 |
* option relevant to Turkic languages 'az' and 'tr'; check for those |
| 706 |
* languages to enable the option. |
| 707 |
*/ |
| 708 |
status = U_ZERO_ERROR; |
| 709 |
uloc_getLanguage(locale, lang, ULOC_LANG_CAPACITY, &status); |
| 710 |
if (U_SUCCESS(status) && status != U_STRING_NOT_TERMINATED_WARNING) |
| 711 |
{ |
| 712 |
/* |
| 713 |
* The option name is confusing, but it causes u_strFoldCase to use |
| 714 |
* the 'T' mappings, which are ignored for U_FOLD_CASE_DEFAULT. |
| 715 |
*/ |
| 716 |
if (strcmp(lang, "tr") == 0 || strcmp(lang, "az") == 0) |
| 717 |
options = U_FOLD_CASE_EXCLUDE_SPECIAL_I; |
| 718 |
} |
| 719 |
|
| 720 |
return u_strFoldCase(dest, destCapacity, src, srcLength, |
| 721 |
options, pErrorCode); |
| 722 |
} |
| 723 |
|
| 724 |
/* |
| 725 |
* strncoll_icu |
| 726 |
* |
| 727 |
* Convert the arguments from the database encoding to UChar strings, then |
| 728 |
* call ucol_strcoll(). An argument length of -1 means that the string is |
| 729 |
* NUL-terminated. |
| 730 |
* |
| 731 |
* When the database encoding is UTF-8, and ICU supports ucol_strcollUTF8(), |
| 732 |
* caller should call that instead. |
| 733 |
*/ |
| 734 |
static int |
| 735 |
strncoll_icu(const char *arg1, ssize_t len1, |
| 736 |
const char *arg2, ssize_t len2, pg_locale_t locale) |
| 737 |
{ |
| 738 |
UChar sbuf[TEXTBUFLEN / sizeof(UChar)]; |
| 739 |
UChar *buf = sbuf; |
| 740 |
int32_t ulen1; |
| 741 |
int32_t ulen2; |
| 742 |
size_t bufsize; |
| 743 |
UChar *uchar1, |
| 744 |
*uchar2; |
| 745 |
int result; |
| 746 |
|
| 747 |
Assert(locale->provider == COLLPROVIDER_ICU); |
| 748 |
|
| 749 |
/* if encoding is UTF8, use more efficient strncoll_icu_utf8 */ |
| 750 |
#ifdef HAVE_UCOL_STRCOLLUTF8 |
| 751 |
Assert(GetDatabaseEncoding() != PG_UTF8); |
| 752 |
#endif |
| 753 |
|
| 754 |
init_icu_converter(); |
| 755 |
|
| 756 |
ulen1 = uchar_length(icu_converter, arg1, len1); |
| 757 |
ulen2 = uchar_length(icu_converter, arg2, len2); |
| 758 |
|
| 759 |
/* ulen1+1 or ulen2+1 doesn't risk overflow, but summing them might */ |
| 760 |
bufsize = add_size(ulen1 + 1, ulen2 + 1); |
| 761 |
if (bufsize > lengthof(sbuf)) |
| 762 |
buf = palloc_array(UChar, bufsize); |
| 763 |
|
| 764 |
uchar1 = buf; |
| 765 |
uchar2 = buf + ulen1 + 1; |
| 766 |
|
| 767 |
ulen1 = uchar_convert(icu_converter, uchar1, ulen1 + 1, arg1, len1); |
| 768 |
ulen2 = uchar_convert(icu_converter, uchar2, ulen2 + 1, arg2, len2); |
| 769 |
|
| 770 |
result = ucol_strcoll(locale->info.icu.ucol, |
| 771 |
uchar1, ulen1, |
| 772 |
uchar2, ulen2); |
| 773 |
|
| 774 |
if (buf != sbuf) |
| 775 |
pfree(buf); |
| 776 |
|
| 777 |
return result; |
| 778 |
} |
| 779 |
|
| 780 |
/* 'srclen' of -1 means the strings are NUL-terminated */ |
| 781 |
static size_t |
| 782 |
strnxfrm_prefix_icu(char *dest, size_t destsize, |
| 783 |
const char *src, ssize_t srclen, |
| 784 |
pg_locale_t locale) |
| 785 |
{ |
| 786 |
UChar sbuf[TEXTBUFLEN / sizeof(UChar)]; |
| 787 |
UChar *uchar = sbuf; |
| 788 |
UCharIterator iter; |
| 789 |
uint32_t state[2]; |
| 790 |
UErrorCode status; |
| 791 |
int32_t ulen; |
| 792 |
Size result_bsize; |
| 793 |
|
| 794 |
Assert(locale->provider == COLLPROVIDER_ICU); |
| 795 |
|
| 796 |
/* if encoding is UTF8, use more efficient strnxfrm_prefix_icu_utf8 */ |
| 797 |
Assert(GetDatabaseEncoding() != PG_UTF8); |
| 798 |
|
| 799 |
init_icu_converter(); |
| 800 |
|
| 801 |
ulen = uchar_length(icu_converter, src, srclen); |
| 802 |
|
| 803 |
if (ulen >= lengthof(sbuf)) |
| 804 |
uchar = palloc_array(UChar, ulen + 1); |
| 805 |
|
| 806 |
ulen = uchar_convert(icu_converter, uchar, ulen + 1, src, srclen); |
| 807 |
|
| 808 |
uiter_setString(&iter, uchar, ulen); |
| 809 |
state[0] = state[1] = 0; /* won't need that again */ |
| 810 |
status = U_ZERO_ERROR; |
| 811 |
result_bsize = ucol_nextSortKeyPart(locale->info.icu.ucol, |
| 812 |
&iter, |
| 813 |
state, |
| 814 |
(uint8_t *) dest, |
| 815 |
destsize, |
| 816 |
&status); |
| 817 |
if (U_FAILURE(status)) |
| 818 |
ereport(ERROR, |
| 819 |
(errmsg("sort key generation failed: %s", |
| 820 |
u_errorName(status)))); |
| 821 |
|
| 822 |
if (uchar != sbuf) |
| 823 |
pfree(uchar); |
| 824 |
|
| 825 |
return result_bsize; |
| 826 |
} |
| 827 |
|
| 828 |
static void |
| 829 |
init_icu_converter(void) |
| 830 |
{ |
| 831 |
const char *icu_encoding_name; |
| 832 |
UErrorCode status; |
| 833 |
UConverter *conv; |
| 834 |
|
| 835 |
if (icu_converter) |
| 836 |
return; /* already done */ |
| 837 |
|
| 838 |
icu_encoding_name = get_encoding_name_for_icu(GetDatabaseEncoding()); |
| 839 |
if (!icu_encoding_name) |
| 840 |
ereport(ERROR, |
| 841 |
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED), |
| 842 |
errmsg("encoding \"%s\" not supported by ICU", |
| 843 |
pg_encoding_to_char(GetDatabaseEncoding())))); |
| 844 |
|
| 845 |
status = U_ZERO_ERROR; |
| 846 |
conv = ucnv_open(icu_encoding_name, &status); |
| 847 |
if (U_FAILURE(status)) |
| 848 |
ereport(ERROR, |
| 849 |
(errmsg("could not open ICU converter for encoding \"%s\": %s", |
| 850 |
icu_encoding_name, u_errorName(status)))); |
| 851 |
|
| 852 |
icu_converter = conv; |
| 853 |
} |
| 854 |
|
| 855 |
/* |
| 856 |
* Find length, in UChars, of given string if converted to UChar string. |
| 857 |
* |
| 858 |
* A length of -1 indicates that the input string is NUL-terminated. |
| 859 |
* |
| 860 |
* Note: given the assumption that the input string fits in MaxAllocSize, |
| 861 |
* the result cannot overflow int32_t. But callers must be careful about |
| 862 |
* multiplying the result by sizeof(UChar). |
| 863 |
*/ |
| 864 |
static int32_t |
| 865 |
uchar_length(UConverter *converter, const char *str, int32_t len) |
| 866 |
{ |
| 867 |
UErrorCode status = U_ZERO_ERROR; |
| 868 |
int32_t ulen; |
| 869 |
|
| 870 |
ulen = ucnv_toUChars(converter, NULL, 0, str, len, &status); |
| 871 |
if (U_FAILURE(status) && status != U_BUFFER_OVERFLOW_ERROR) |
| 872 |
ereport(ERROR, |
| 873 |
(errmsg("%s failed: %s", "ucnv_toUChars", u_errorName(status)))); |
| 874 |
return ulen; |
| 875 |
} |
| 876 |
|
| 877 |
/* |
| 878 |
* Convert the given source string into a UChar string, stored in dest, and |
| 879 |
* return the length (in UChars). |
| 880 |
* |
| 881 |
* A srclen of -1 indicates that the input string is NUL-terminated. |
| 882 |
*/ |
| 883 |
static int32_t |
| 884 |
uchar_convert(UConverter *converter, UChar *dest, int32_t destlen, |
| 885 |
const char *src, int32_t srclen) |
| 886 |
{ |
| 887 |
UErrorCode status = U_ZERO_ERROR; |
| 888 |
int32_t ulen; |
| 889 |
|
| 890 |
ulen = ucnv_toUChars(converter, dest, destlen, src, srclen, &status); |
| 891 |
if (U_FAILURE(status)) |
| 892 |
ereport(ERROR, |
| 893 |
(errmsg("%s failed: %s", "ucnv_toUChars", u_errorName(status)))); |
| 894 |
return ulen; |
| 895 |
} |
| 896 |
|
| 897 |
/* |
| 898 |
* Parse collation attributes from the given locale string and apply them to |
| 899 |
* the open collator. |
| 900 |
* |
| 901 |
* First, the locale string is canonicalized to an ICU format locale ID such |
| 902 |
* as "und@colStrength=primary;colCaseLevel=yes". Then, it parses and applies |
| 903 |
* the key-value arguments. |
| 904 |
* |
| 905 |
* Starting with ICU version 54, the attributes are processed automatically by |
| 906 |
* ucol_open(), so this is only necessary for emulating this behavior on older |
| 907 |
* versions. |
| 908 |
*/ |
| 909 |
pg_attribute_unused() |
| 910 |
static void |
| 911 |
icu_set_collation_attributes(UCollator *collator, const char *loc, |
| 912 |
UErrorCode *status) |
| 913 |
{ |
| 914 |
int32_t len; |
| 915 |
char *icu_locale_id; |
| 916 |
char *lower_str; |
| 917 |
char *str; |
| 918 |
char *token; |
| 919 |
|
| 920 |
/* |
| 921 |
* The input locale may be a BCP 47 language tag, e.g. |
| 922 |
* "und-u-kc-ks-level1", which expresses the same attributes in a |
| 923 |
* different form. It will be converted to the equivalent ICU format |
| 924 |
* locale ID, e.g. "und@colcaselevel=yes;colstrength=primary", by |
| 925 |
* uloc_canonicalize(). |
| 926 |
*/ |
| 927 |
*status = U_ZERO_ERROR; |
| 928 |
len = uloc_canonicalize(loc, NULL, 0, status); |
| 929 |
icu_locale_id = palloc(len + 1); |
| 930 |
*status = U_ZERO_ERROR; |
| 931 |
len = uloc_canonicalize(loc, icu_locale_id, len + 1, status); |
| 932 |
if (U_FAILURE(*status) || *status == U_STRING_NOT_TERMINATED_WARNING) |
| 933 |
return; |
| 934 |
|
| 935 |
lower_str = asc_tolower(icu_locale_id, strlen(icu_locale_id)); |
| 936 |
|
| 937 |
pfree(icu_locale_id); |
| 938 |
|
| 939 |
str = strchr(lower_str, '@'); |
| 940 |
if (!str) |
| 941 |
return; |
| 942 |
str++; |
| 943 |
|
| 944 |
while ((token = strsep(&str, ";"))) |
| 945 |
{ |
| 946 |
char *e = strchr(token, '='); |
| 947 |
|
| 948 |
if (e) |
| 949 |
{ |
| 950 |
char *name; |
| 951 |
char *value; |
| 952 |
UColAttribute uattr; |
| 953 |
UColAttributeValue uvalue; |
| 954 |
|
| 955 |
*status = U_ZERO_ERROR; |
| 956 |
|
| 957 |
*e = '\0'; |
| 958 |
name = token; |
| 959 |
value = e + 1; |
| 960 |
|
| 961 |
/* |
| 962 |
* See attribute name and value lists in ICU i18n/coll.cpp |
| 963 |
*/ |
| 964 |
if (strcmp(name, "colstrength") == 0) |
| 965 |
uattr = UCOL_STRENGTH; |
| 966 |
else if (strcmp(name, "colbackwards") == 0) |
| 967 |
uattr = UCOL_FRENCH_COLLATION; |
| 968 |
else if (strcmp(name, "colcaselevel") == 0) |
| 969 |
uattr = UCOL_CASE_LEVEL; |
| 970 |
else if (strcmp(name, "colcasefirst") == 0) |
| 971 |
uattr = UCOL_CASE_FIRST; |
| 972 |
else if (strcmp(name, "colalternate") == 0) |
| 973 |
uattr = UCOL_ALTERNATE_HANDLING; |
| 974 |
else if (strcmp(name, "colnormalization") == 0) |
| 975 |
uattr = UCOL_NORMALIZATION_MODE; |
| 976 |
else if (strcmp(name, "colnumeric") == 0) |
| 977 |
uattr = UCOL_NUMERIC_COLLATION; |
| 978 |
else |
| 979 |
/* ignore if unknown */ |
| 980 |
continue; |
| 981 |
|
| 982 |
if (strcmp(value, "primary") == 0) |
| 983 |
uvalue = UCOL_PRIMARY; |
| 984 |
else if (strcmp(value, "secondary") == 0) |
| 985 |
uvalue = UCOL_SECONDARY; |
| 986 |
else if (strcmp(value, "tertiary") == 0) |
| 987 |
uvalue = UCOL_TERTIARY; |
| 988 |
else if (strcmp(value, "quaternary") == 0) |
| 989 |
uvalue = UCOL_QUATERNARY; |
| 990 |
else if (strcmp(value, "identical") == 0) |
| 991 |
uvalue = UCOL_IDENTICAL; |
| 992 |
else if (strcmp(value, "no") == 0) |
| 993 |
uvalue = UCOL_OFF; |
| 994 |
else if (strcmp(value, "yes") == 0) |
| 995 |
uvalue = UCOL_ON; |
| 996 |
else if (strcmp(value, "shifted") == 0) |
| 997 |
uvalue = UCOL_SHIFTED; |
| 998 |
else if (strcmp(value, "non-ignorable") == 0) |
| 999 |
uvalue = UCOL_NON_IGNORABLE; |
| 1000 |
else if (strcmp(value, "lower") == 0) |
| 1001 |
uvalue = UCOL_LOWER_FIRST; |
| 1002 |
else if (strcmp(value, "upper") == 0) |
| 1003 |
uvalue = UCOL_UPPER_FIRST; |
| 1004 |
else |
| 1005 |
{ |
| 1006 |
*status = U_ILLEGAL_ARGUMENT_ERROR; |
| 1007 |
break; |
| 1008 |
} |
| 1009 |
|
| 1010 |
ucol_setAttribute(collator, uattr, uvalue, status); |
| 1011 |
} |
| 1012 |
} |
| 1013 |
|
| 1014 |
pfree(lower_str); |
| 1015 |
} |
| 1016 |
|
| 1017 |
#endif /* USE_ICU */ |