| 1 |
.\" -*- mode: troff; coding: utf-8 -*- |
| 2 |
.\" Automatically generated by Pod::Man 5.0102 (Pod::Simple 3.45) |
| 3 |
.\" |
| 4 |
.\" Standard preamble: |
| 5 |
.\" ======================================================================== |
| 6 |
.de Sp \" Vertical space (when we can't use .PP) |
| 7 |
.if t .sp .5v |
| 8 |
.if n .sp |
| 9 |
.. |
| 10 |
.de Vb \" Begin verbatim text |
| 11 |
.ft CW |
| 12 |
.nf |
| 13 |
.ne \\$1 |
| 14 |
.. |
| 15 |
.de Ve \" End verbatim text |
| 16 |
.ft R |
| 17 |
.fi |
| 18 |
.. |
| 19 |
.\" \*(C` and \*(C' are quotes in nroff, nothing in troff, for use with C<>. |
| 20 |
.ie n \{\ |
| 21 |
. ds C` "" |
| 22 |
. ds C' "" |
| 23 |
'br\} |
| 24 |
.el\{\ |
| 25 |
. ds C` |
| 26 |
. ds C' |
| 27 |
'br\} |
| 28 |
.\" |
| 29 |
.\" Escape single quotes in literal strings from groff's Unicode transform. |
| 30 |
.ie \n(.g .ds Aq \(aq |
| 31 |
.el .ds Aq ' |
| 32 |
.\" |
| 33 |
.\" If the F register is >0, we'll generate index entries on stderr for |
| 34 |
.\" titles (.TH), headers (.SH), subsections (.SS), items (.Ip), and index |
| 35 |
.\" entries marked with X<> in POD. Of course, you'll have to process the |
| 36 |
.\" output yourself in some meaningful fashion. |
| 37 |
.\" |
| 38 |
.\" Avoid warning from groff about undefined register 'F'. |
| 39 |
.de IX |
| 40 |
.. |
| 41 |
.nr rF 0 |
| 42 |
.if \n(.g .if rF .nr rF 1 |
| 43 |
.if (\n(rF:(\n(.g==0)) \{\ |
| 44 |
. if \nF \{\ |
| 45 |
. de IX |
| 46 |
. tm Index:\\$1\t\\n%\t"\\$2" |
| 47 |
.. |
| 48 |
. if !\nF==2 \{\ |
| 49 |
. nr % 0 |
| 50 |
. nr F 2 |
| 51 |
. \} |
| 52 |
. \} |
| 53 |
.\} |
| 54 |
.rr rF |
| 55 |
.\" ======================================================================== |
| 56 |
.\" |
| 57 |
.IX Title "Unicode::Normalize 3perl" |
| 58 |
.TH Unicode::Normalize 3perl 2025-11-16 "perl v5.40.1" "Perl Programmers Reference Guide" |
| 59 |
.\" For nroff, turn off justification. Always turn off hyphenation; it makes |
| 60 |
.\" way too many mistakes in technical documents. |
| 61 |
.if n .ad l |
| 62 |
.nh |
| 63 |
.SH NAME |
| 64 |
Unicode::Normalize \- Unicode Normalization Forms |
| 65 |
.SH SYNOPSIS |
| 66 |
.IX Header "SYNOPSIS" |
| 67 |
(1) using function names exported by default: |
| 68 |
.PP |
| 69 |
.Vb 1 |
| 70 |
\& use Unicode::Normalize; |
| 71 |
\& |
| 72 |
\& $NFD_string = NFD($string); # Normalization Form D |
| 73 |
\& $NFC_string = NFC($string); # Normalization Form C |
| 74 |
\& $NFKD_string = NFKD($string); # Normalization Form KD |
| 75 |
\& $NFKC_string = NFKC($string); # Normalization Form KC |
| 76 |
.Ve |
| 77 |
.PP |
| 78 |
(2) using function names exported on request: |
| 79 |
.PP |
| 80 |
.Vb 1 |
| 81 |
\& use Unicode::Normalize \*(Aqnormalize\*(Aq; |
| 82 |
\& |
| 83 |
\& $NFD_string = normalize(\*(AqD\*(Aq, $string); # Normalization Form D |
| 84 |
\& $NFC_string = normalize(\*(AqC\*(Aq, $string); # Normalization Form C |
| 85 |
\& $NFKD_string = normalize(\*(AqKD\*(Aq, $string); # Normalization Form KD |
| 86 |
\& $NFKC_string = normalize(\*(AqKC\*(Aq, $string); # Normalization Form KC |
| 87 |
.Ve |
| 88 |
.SH DESCRIPTION |
| 89 |
.IX Header "DESCRIPTION" |
| 90 |
Parameters: |
| 91 |
.PP |
| 92 |
\&\f(CW$string\fR is used as a string under character semantics (see perlunicode). |
| 93 |
.PP |
| 94 |
\&\f(CW$code_point\fR should be an unsigned integer representing a Unicode code point. |
| 95 |
.PP |
| 96 |
Note: Between XSUB and pure Perl, there is an incompatibility |
| 97 |
about the interpretation of \f(CW$code_point\fR as a decimal number. |
| 98 |
XSUB converts \f(CW$code_point\fR to an unsigned integer, but pure Perl does not. |
| 99 |
Do not use a floating point nor a negative sign in \f(CW$code_point\fR. |
| 100 |
.SS "Normalization Forms" |
| 101 |
.IX Subsection "Normalization Forms" |
| 102 |
.ie n .IP """$NFD_string = NFD($string)""" 4 |
| 103 |
.el .IP "\f(CW$NFD_string = NFD($string)\fR" 4 |
| 104 |
.IX Item "$NFD_string = NFD($string)" |
| 105 |
It returns the Normalization Form D (formed by canonical decomposition). |
| 106 |
.ie n .IP """$NFC_string = NFC($string)""" 4 |
| 107 |
.el .IP "\f(CW$NFC_string = NFC($string)\fR" 4 |
| 108 |
.IX Item "$NFC_string = NFC($string)" |
| 109 |
It returns the Normalization Form C (formed by canonical decomposition |
| 110 |
followed by canonical composition). |
| 111 |
.ie n .IP """$NFKD_string = NFKD($string)""" 4 |
| 112 |
.el .IP "\f(CW$NFKD_string = NFKD($string)\fR" 4 |
| 113 |
.IX Item "$NFKD_string = NFKD($string)" |
| 114 |
It returns the Normalization Form KD (formed by compatibility decomposition). |
| 115 |
.ie n .IP """$NFKC_string = NFKC($string)""" 4 |
| 116 |
.el .IP "\f(CW$NFKC_string = NFKC($string)\fR" 4 |
| 117 |
.IX Item "$NFKC_string = NFKC($string)" |
| 118 |
It returns the Normalization Form KC (formed by compatibility decomposition |
| 119 |
followed by \fBcanonical\fR composition). |
| 120 |
.ie n .IP """$FCD_string = FCD($string)""" 4 |
| 121 |
.el .IP "\f(CW$FCD_string = FCD($string)\fR" 4 |
| 122 |
.IX Item "$FCD_string = FCD($string)" |
| 123 |
If the given string is in FCD ("Fast C or D" form; cf. UTN #5), |
| 124 |
it returns the string without modification; otherwise it returns an FCD string. |
| 125 |
.Sp |
| 126 |
Note: FCD is not always unique, then plural forms may be equivalent |
| 127 |
each other. \f(CWFCD()\fR will return one of these equivalent forms. |
| 128 |
.ie n .IP """$FCC_string = FCC($string)""" 4 |
| 129 |
.el .IP "\f(CW$FCC_string = FCC($string)\fR" 4 |
| 130 |
.IX Item "$FCC_string = FCC($string)" |
| 131 |
It returns the FCC form ("Fast C Contiguous"; cf. UTN #5). |
| 132 |
.Sp |
| 133 |
Note: FCC is unique, as well as four normalization forms (NF*). |
| 134 |
.ie n .IP """$normalized_string = normalize($form_name, $string)""" 4 |
| 135 |
.el .IP "\f(CW$normalized_string = normalize($form_name, $string)\fR" 4 |
| 136 |
.IX Item "$normalized_string = normalize($form_name, $string)" |
| 137 |
It returns the normalization form of \f(CW$form_name\fR. |
| 138 |
.Sp |
| 139 |
As \f(CW$form_name\fR, one of the following names must be given. |
| 140 |
.Sp |
| 141 |
.Vb 4 |
| 142 |
\& \*(AqC\*(Aq or \*(AqNFC\*(Aq for Normalization Form C (UAX #15) |
| 143 |
\& \*(AqD\*(Aq or \*(AqNFD\*(Aq for Normalization Form D (UAX #15) |
| 144 |
\& \*(AqKC\*(Aq or \*(AqNFKC\*(Aq for Normalization Form KC (UAX #15) |
| 145 |
\& \*(AqKD\*(Aq or \*(AqNFKD\*(Aq for Normalization Form KD (UAX #15) |
| 146 |
\& |
| 147 |
\& \*(AqFCD\*(Aq for "Fast C or D" Form (UTN #5) |
| 148 |
\& \*(AqFCC\*(Aq for "Fast C Contiguous" (UTN #5) |
| 149 |
.Ve |
| 150 |
.SS "Decomposition and Composition" |
| 151 |
.IX Subsection "Decomposition and Composition" |
| 152 |
.ie n .IP """$decomposed_string = decompose($string [, $useCompatMapping])""" 4 |
| 153 |
.el .IP "\f(CW$decomposed_string = decompose($string [, $useCompatMapping])\fR" 4 |
| 154 |
.IX Item "$decomposed_string = decompose($string [, $useCompatMapping])" |
| 155 |
It returns the concatenation of the decomposition of each character |
| 156 |
in the string. |
| 157 |
.Sp |
| 158 |
If the second parameter (a boolean) is omitted or false, |
| 159 |
the decomposition is canonical decomposition; |
| 160 |
if the second parameter (a boolean) is true, |
| 161 |
the decomposition is compatibility decomposition. |
| 162 |
.Sp |
| 163 |
The string returned is not always in NFD/NFKD. Reordering may be required. |
| 164 |
.Sp |
| 165 |
.Vb 2 |
| 166 |
\& $NFD_string = reorder(decompose($string)); # eq. to NFD() |
| 167 |
\& $NFKD_string = reorder(decompose($string, TRUE)); # eq. to NFKD() |
| 168 |
.Ve |
| 169 |
.ie n .IP """$reordered_string = reorder($string)""" 4 |
| 170 |
.el .IP "\f(CW$reordered_string = reorder($string)\fR" 4 |
| 171 |
.IX Item "$reordered_string = reorder($string)" |
| 172 |
It returns the result of reordering the combining characters |
| 173 |
according to Canonical Ordering Behavior. |
| 174 |
.Sp |
| 175 |
For example, when you have a list of NFD/NFKD strings, |
| 176 |
you can get the concatenated NFD/NFKD string from them, by saying |
| 177 |
.Sp |
| 178 |
.Vb 2 |
| 179 |
\& $concat_NFD = reorder(join \*(Aq\*(Aq, @NFD_strings); |
| 180 |
\& $concat_NFKD = reorder(join \*(Aq\*(Aq, @NFKD_strings); |
| 181 |
.Ve |
| 182 |
.ie n .IP """$composed_string = compose($string)""" 4 |
| 183 |
.el .IP "\f(CW$composed_string = compose($string)\fR" 4 |
| 184 |
.IX Item "$composed_string = compose($string)" |
| 185 |
It returns the result of canonical composition |
| 186 |
without applying any decomposition. |
| 187 |
.Sp |
| 188 |
For example, when you have a NFD/NFKD string, |
| 189 |
you can get its NFC/NFKC string, by saying |
| 190 |
.Sp |
| 191 |
.Vb 2 |
| 192 |
\& $NFC_string = compose($NFD_string); |
| 193 |
\& $NFKC_string = compose($NFKD_string); |
| 194 |
.Ve |
| 195 |
.ie n .IP """($processed, $unprocessed) = splitOnLastStarter($normalized)""" 4 |
| 196 |
.el .IP "\f(CW($processed, $unprocessed) = splitOnLastStarter($normalized)\fR" 4 |
| 197 |
.IX Item "($processed, $unprocessed) = splitOnLastStarter($normalized)" |
| 198 |
It returns two strings: the first one, \f(CW$processed\fR, is a part |
| 199 |
before the last starter, and the second one, \f(CW$unprocessed\fR is |
| 200 |
another part after the first part. A starter is a character having |
| 201 |
a combining class of zero (see UAX #15). |
| 202 |
.Sp |
| 203 |
Note that \f(CW$processed\fR may be empty (when \f(CW$normalized\fR contains no |
| 204 |
starter or starts with the last starter), and then \f(CW$unprocessed\fR |
| 205 |
should be equal to the entire \f(CW$normalized\fR. |
| 206 |
.Sp |
| 207 |
When you have a \f(CW$normalized\fR string and an \f(CW$unnormalized\fR string |
| 208 |
following it, a simple concatenation is wrong: |
| 209 |
.Sp |
| 210 |
.Vb 1 |
| 211 |
\& $concat = $normalized . normalize($form, $unnormalized); # wrong! |
| 212 |
.Ve |
| 213 |
.Sp |
| 214 |
Instead of it, do like this: |
| 215 |
.Sp |
| 216 |
.Vb 2 |
| 217 |
\& ($processed, $unprocessed) = splitOnLastStarter($normalized); |
| 218 |
\& $concat = $processed . normalize($form,$unprocessed.$unnormalized); |
| 219 |
.Ve |
| 220 |
.Sp |
| 221 |
\&\f(CWsplitOnLastStarter()\fR should be called with a pre-normalized parameter |
| 222 |
\&\f(CW$normalized\fR, that is in the same form as \f(CW$form\fR you want. |
| 223 |
.Sp |
| 224 |
If you have an array of \f(CW@string\fR that should be concatenated and then |
| 225 |
normalized, you can do like this: |
| 226 |
.Sp |
| 227 |
.Vb 11 |
| 228 |
\& my $result = ""; |
| 229 |
\& my $unproc = ""; |
| 230 |
\& foreach my $str (@string) { |
| 231 |
\& $unproc .= $str; |
| 232 |
\& my $n = normalize($form, $unproc); |
| 233 |
\& my($p, $u) = splitOnLastStarter($n); |
| 234 |
\& $result .= $p; |
| 235 |
\& $unproc = $u; |
| 236 |
\& } |
| 237 |
\& $result .= $unproc; |
| 238 |
\& # instead of normalize($form, join(\*(Aq\*(Aq, @string)) |
| 239 |
.Ve |
| 240 |
.ie n .IP """$processed = normalize_partial($form, $unprocessed)""" 4 |
| 241 |
.el .IP "\f(CW$processed = normalize_partial($form, $unprocessed)\fR" 4 |
| 242 |
.IX Item "$processed = normalize_partial($form, $unprocessed)" |
| 243 |
A wrapper for the combination of \f(CWnormalize()\fR and \f(CWsplitOnLastStarter()\fR. |
| 244 |
Note that \f(CW$unprocessed\fR will be modified as a side-effect. |
| 245 |
.Sp |
| 246 |
If you have an array of \f(CW@string\fR that should be concatenated and then |
| 247 |
normalized, you can do like this: |
| 248 |
.Sp |
| 249 |
.Vb 8 |
| 250 |
\& my $result = ""; |
| 251 |
\& my $unproc = ""; |
| 252 |
\& foreach my $str (@string) { |
| 253 |
\& $unproc .= $str; |
| 254 |
\& $result .= normalize_partial($form, $unproc); |
| 255 |
\& } |
| 256 |
\& $result .= $unproc; |
| 257 |
\& # instead of normalize($form, join(\*(Aq\*(Aq, @string)) |
| 258 |
.Ve |
| 259 |
.ie n .IP """$processed = NFD_partial($unprocessed)""" 4 |
| 260 |
.el .IP "\f(CW$processed = NFD_partial($unprocessed)\fR" 4 |
| 261 |
.IX Item "$processed = NFD_partial($unprocessed)" |
| 262 |
It does like \f(CW\*(C`normalize_partial(\*(AqNFD\*(Aq, $unprocessed)\*(C'\fR. |
| 263 |
Note that \f(CW$unprocessed\fR will be modified as a side-effect. |
| 264 |
.ie n .IP """$processed = NFC_partial($unprocessed)""" 4 |
| 265 |
.el .IP "\f(CW$processed = NFC_partial($unprocessed)\fR" 4 |
| 266 |
.IX Item "$processed = NFC_partial($unprocessed)" |
| 267 |
It does like \f(CW\*(C`normalize_partial(\*(AqNFC\*(Aq, $unprocessed)\*(C'\fR. |
| 268 |
Note that \f(CW$unprocessed\fR will be modified as a side-effect. |
| 269 |
.ie n .IP """$processed = NFKD_partial($unprocessed)""" 4 |
| 270 |
.el .IP "\f(CW$processed = NFKD_partial($unprocessed)\fR" 4 |
| 271 |
.IX Item "$processed = NFKD_partial($unprocessed)" |
| 272 |
It does like \f(CW\*(C`normalize_partial(\*(AqNFKD\*(Aq, $unprocessed)\*(C'\fR. |
| 273 |
Note that \f(CW$unprocessed\fR will be modified as a side-effect. |
| 274 |
.ie n .IP """$processed = NFKC_partial($unprocessed)""" 4 |
| 275 |
.el .IP "\f(CW$processed = NFKC_partial($unprocessed)\fR" 4 |
| 276 |
.IX Item "$processed = NFKC_partial($unprocessed)" |
| 277 |
It does like \f(CW\*(C`normalize_partial(\*(AqNFKC\*(Aq, $unprocessed)\*(C'\fR. |
| 278 |
Note that \f(CW$unprocessed\fR will be modified as a side-effect. |
| 279 |
.SS "Quick Check" |
| 280 |
.IX Subsection "Quick Check" |
| 281 |
(see Annex 8, UAX #15; and \fIlib/unicore/DerivedNormalizationProps.txt\fR) |
| 282 |
.PP |
| 283 |
The following functions check whether the string is in that normalization form. |
| 284 |
.PP |
| 285 |
The result returned will be one of the following: |
| 286 |
.PP |
| 287 |
.Vb 3 |
| 288 |
\& YES The string is in that normalization form. |
| 289 |
\& NO The string is not in that normalization form. |
| 290 |
\& MAYBE Dubious. Maybe yes, maybe no. |
| 291 |
.Ve |
| 292 |
.ie n .IP """$result = checkNFD($string)""" 4 |
| 293 |
.el .IP "\f(CW$result = checkNFD($string)\fR" 4 |
| 294 |
.IX Item "$result = checkNFD($string)" |
| 295 |
It returns true (\f(CW1\fR) if \f(CW\*(C`YES\*(C'\fR; false (\f(CW\*(C`empty string\*(C'\fR) if \f(CW\*(C`NO\*(C'\fR. |
| 296 |
.ie n .IP """$result = checkNFC($string)""" 4 |
| 297 |
.el .IP "\f(CW$result = checkNFC($string)\fR" 4 |
| 298 |
.IX Item "$result = checkNFC($string)" |
| 299 |
It returns true (\f(CW1\fR) if \f(CW\*(C`YES\*(C'\fR; false (\f(CW\*(C`empty string\*(C'\fR) if \f(CW\*(C`NO\*(C'\fR; |
| 300 |
\&\f(CW\*(C`undef\*(C'\fR if \f(CW\*(C`MAYBE\*(C'\fR. |
| 301 |
.ie n .IP """$result = checkNFKD($string)""" 4 |
| 302 |
.el .IP "\f(CW$result = checkNFKD($string)\fR" 4 |
| 303 |
.IX Item "$result = checkNFKD($string)" |
| 304 |
It returns true (\f(CW1\fR) if \f(CW\*(C`YES\*(C'\fR; false (\f(CW\*(C`empty string\*(C'\fR) if \f(CW\*(C`NO\*(C'\fR. |
| 305 |
.ie n .IP """$result = checkNFKC($string)""" 4 |
| 306 |
.el .IP "\f(CW$result = checkNFKC($string)\fR" 4 |
| 307 |
.IX Item "$result = checkNFKC($string)" |
| 308 |
It returns true (\f(CW1\fR) if \f(CW\*(C`YES\*(C'\fR; false (\f(CW\*(C`empty string\*(C'\fR) if \f(CW\*(C`NO\*(C'\fR; |
| 309 |
\&\f(CW\*(C`undef\*(C'\fR if \f(CW\*(C`MAYBE\*(C'\fR. |
| 310 |
.ie n .IP """$result = checkFCD($string)""" 4 |
| 311 |
.el .IP "\f(CW$result = checkFCD($string)\fR" 4 |
| 312 |
.IX Item "$result = checkFCD($string)" |
| 313 |
It returns true (\f(CW1\fR) if \f(CW\*(C`YES\*(C'\fR; false (\f(CW\*(C`empty string\*(C'\fR) if \f(CW\*(C`NO\*(C'\fR. |
| 314 |
.ie n .IP """$result = checkFCC($string)""" 4 |
| 315 |
.el .IP "\f(CW$result = checkFCC($string)\fR" 4 |
| 316 |
.IX Item "$result = checkFCC($string)" |
| 317 |
It returns true (\f(CW1\fR) if \f(CW\*(C`YES\*(C'\fR; false (\f(CW\*(C`empty string\*(C'\fR) if \f(CW\*(C`NO\*(C'\fR; |
| 318 |
\&\f(CW\*(C`undef\*(C'\fR if \f(CW\*(C`MAYBE\*(C'\fR. |
| 319 |
.Sp |
| 320 |
Note: If a string is not in FCD, it must not be in FCC. |
| 321 |
So \f(CWcheckFCC($not_FCD_string)\fR should return \f(CW\*(C`NO\*(C'\fR. |
| 322 |
.ie n .IP """$result = check($form_name, $string)""" 4 |
| 323 |
.el .IP "\f(CW$result = check($form_name, $string)\fR" 4 |
| 324 |
.IX Item "$result = check($form_name, $string)" |
| 325 |
It returns true (\f(CW1\fR) if \f(CW\*(C`YES\*(C'\fR; false (\f(CW\*(C`empty string\*(C'\fR) if \f(CW\*(C`NO\*(C'\fR; |
| 326 |
\&\f(CW\*(C`undef\*(C'\fR if \f(CW\*(C`MAYBE\*(C'\fR. |
| 327 |
.Sp |
| 328 |
As \f(CW$form_name\fR, one of the following names must be given. |
| 329 |
.Sp |
| 330 |
.Vb 4 |
| 331 |
\& \*(AqC\*(Aq or \*(AqNFC\*(Aq for Normalization Form C (UAX #15) |
| 332 |
\& \*(AqD\*(Aq or \*(AqNFD\*(Aq for Normalization Form D (UAX #15) |
| 333 |
\& \*(AqKC\*(Aq or \*(AqNFKC\*(Aq for Normalization Form KC (UAX #15) |
| 334 |
\& \*(AqKD\*(Aq or \*(AqNFKD\*(Aq for Normalization Form KD (UAX #15) |
| 335 |
\& |
| 336 |
\& \*(AqFCD\*(Aq for "Fast C or D" Form (UTN #5) |
| 337 |
\& \*(AqFCC\*(Aq for "Fast C Contiguous" (UTN #5) |
| 338 |
.Ve |
| 339 |
.PP |
| 340 |
\&\fBNote\fR |
| 341 |
.PP |
| 342 |
In the cases of NFD, NFKD, and FCD, the answer must be |
| 343 |
either \f(CW\*(C`YES\*(C'\fR or \f(CW\*(C`NO\*(C'\fR. The answer \f(CW\*(C`MAYBE\*(C'\fR may be returned |
| 344 |
in the cases of NFC, NFKC, and FCC. |
| 345 |
.PP |
| 346 |
A \f(CW\*(C`MAYBE\*(C'\fR string should contain at least one combining character |
| 347 |
or the like. For example, \f(CW\*(C`COMBINING ACUTE ACCENT\*(C'\fR has |
| 348 |
the MAYBE_NFC/MAYBE_NFKC property. |
| 349 |
.PP |
| 350 |
Both \f(CW\*(C`checkNFC("A\eN{COMBINING ACUTE ACCENT}")\*(C'\fR |
| 351 |
and \f(CW\*(C`checkNFC("B\eN{COMBINING ACUTE ACCENT}")\*(C'\fR will return \f(CW\*(C`MAYBE\*(C'\fR. |
| 352 |
\&\f(CW"A\eN{COMBINING ACUTE ACCENT}"\fR is not in NFC |
| 353 |
(its NFC is \f(CW"\eN{LATIN CAPITAL LETTER A WITH ACUTE}"\fR), |
| 354 |
while \f(CW"B\eN{COMBINING ACUTE ACCENT}"\fR is in NFC. |
| 355 |
.PP |
| 356 |
If you want to check exactly, compare the string with its NFC/NFKC/FCC. |
| 357 |
.PP |
| 358 |
.Vb 5 |
| 359 |
\& if ($string eq NFC($string)) { |
| 360 |
\& # $string is exactly normalized in NFC; |
| 361 |
\& } else { |
| 362 |
\& # $string is not normalized in NFC; |
| 363 |
\& } |
| 364 |
\& |
| 365 |
\& if ($string eq NFKC($string)) { |
| 366 |
\& # $string is exactly normalized in NFKC; |
| 367 |
\& } else { |
| 368 |
\& # $string is not normalized in NFKC; |
| 369 |
\& } |
| 370 |
.Ve |
| 371 |
.SS "Character Data" |
| 372 |
.IX Subsection "Character Data" |
| 373 |
These functions are interface of character data used internally. |
| 374 |
If you want only to get Unicode normalization forms, you don't need |
| 375 |
call them yourself. |
| 376 |
.ie n .IP """$canonical_decomposition = getCanon($code_point)""" 4 |
| 377 |
.el .IP "\f(CW$canonical_decomposition = getCanon($code_point)\fR" 4 |
| 378 |
.IX Item "$canonical_decomposition = getCanon($code_point)" |
| 379 |
If the character is canonically decomposable (including Hangul Syllables), |
| 380 |
it returns the (full) canonical decomposition as a string. |
| 381 |
Otherwise it returns \f(CW\*(C`undef\*(C'\fR. |
| 382 |
.Sp |
| 383 |
\&\fBNote:\fR According to the Unicode standard, the canonical decomposition |
| 384 |
of the character that is not canonically decomposable is same as |
| 385 |
the character itself. |
| 386 |
.ie n .IP """$compatibility_decomposition = getCompat($code_point)""" 4 |
| 387 |
.el .IP "\f(CW$compatibility_decomposition = getCompat($code_point)\fR" 4 |
| 388 |
.IX Item "$compatibility_decomposition = getCompat($code_point)" |
| 389 |
If the character is compatibility decomposable (including Hangul Syllables), |
| 390 |
it returns the (full) compatibility decomposition as a string. |
| 391 |
Otherwise it returns \f(CW\*(C`undef\*(C'\fR. |
| 392 |
.Sp |
| 393 |
\&\fBNote:\fR According to the Unicode standard, the compatibility decomposition |
| 394 |
of the character that is not compatibility decomposable is same as |
| 395 |
the character itself. |
| 396 |
.ie n .IP """$code_point_composite = getComposite($code_point_here, $code_point_next)""" 4 |
| 397 |
.el .IP "\f(CW$code_point_composite = getComposite($code_point_here, $code_point_next)\fR" 4 |
| 398 |
.IX Item "$code_point_composite = getComposite($code_point_here, $code_point_next)" |
| 399 |
If two characters here and next (as code points) are composable |
| 400 |
(including Hangul Jamo/Syllables and Composition Exclusions), |
| 401 |
it returns the code point of the composite. |
| 402 |
.Sp |
| 403 |
If they are not composable, it returns \f(CW\*(C`undef\*(C'\fR. |
| 404 |
.ie n .IP """$combining_class = getCombinClass($code_point)""" 4 |
| 405 |
.el .IP "\f(CW$combining_class = getCombinClass($code_point)\fR" 4 |
| 406 |
.IX Item "$combining_class = getCombinClass($code_point)" |
| 407 |
It returns the combining class (as an integer) of the character. |
| 408 |
.ie n .IP """$may_be_composed_with_prev_char = isComp2nd($code_point)""" 4 |
| 409 |
.el .IP "\f(CW$may_be_composed_with_prev_char = isComp2nd($code_point)\fR" 4 |
| 410 |
.IX Item "$may_be_composed_with_prev_char = isComp2nd($code_point)" |
| 411 |
It returns a boolean whether the character of the specified codepoint |
| 412 |
may be composed with the previous one in a certain composition |
| 413 |
(including Hangul Compositions, but excluding |
| 414 |
Composition Exclusions and Non-Starter Decompositions). |
| 415 |
.ie n .IP """$is_exclusion = isExclusion($code_point)""" 4 |
| 416 |
.el .IP "\f(CW$is_exclusion = isExclusion($code_point)\fR" 4 |
| 417 |
.IX Item "$is_exclusion = isExclusion($code_point)" |
| 418 |
It returns a boolean whether the code point is a composition exclusion. |
| 419 |
.ie n .IP """$is_singleton = isSingleton($code_point)""" 4 |
| 420 |
.el .IP "\f(CW$is_singleton = isSingleton($code_point)\fR" 4 |
| 421 |
.IX Item "$is_singleton = isSingleton($code_point)" |
| 422 |
It returns a boolean whether the code point is a singleton |
| 423 |
.ie n .IP """$is_non_starter_decomposition = isNonStDecomp($code_point)""" 4 |
| 424 |
.el .IP "\f(CW$is_non_starter_decomposition = isNonStDecomp($code_point)\fR" 4 |
| 425 |
.IX Item "$is_non_starter_decomposition = isNonStDecomp($code_point)" |
| 426 |
It returns a boolean whether the code point has Non-Starter Decomposition. |
| 427 |
.ie n .IP """$is_Full_Composition_Exclusion = isComp_Ex($code_point)""" 4 |
| 428 |
.el .IP "\f(CW$is_Full_Composition_Exclusion = isComp_Ex($code_point)\fR" 4 |
| 429 |
.IX Item "$is_Full_Composition_Exclusion = isComp_Ex($code_point)" |
| 430 |
It returns a boolean of the derived property Comp_Ex |
| 431 |
(Full_Composition_Exclusion). This property is generated from |
| 432 |
Composition Exclusions + Singletons + Non-Starter Decompositions. |
| 433 |
.ie n .IP """$NFD_is_NO = isNFD_NO($code_point)""" 4 |
| 434 |
.el .IP "\f(CW$NFD_is_NO = isNFD_NO($code_point)\fR" 4 |
| 435 |
.IX Item "$NFD_is_NO = isNFD_NO($code_point)" |
| 436 |
It returns a boolean of the derived property NFD_NO |
| 437 |
(NFD_Quick_Check=No). |
| 438 |
.ie n .IP """$NFC_is_NO = isNFC_NO($code_point)""" 4 |
| 439 |
.el .IP "\f(CW$NFC_is_NO = isNFC_NO($code_point)\fR" 4 |
| 440 |
.IX Item "$NFC_is_NO = isNFC_NO($code_point)" |
| 441 |
It returns a boolean of the derived property NFC_NO |
| 442 |
(NFC_Quick_Check=No). |
| 443 |
.ie n .IP """$NFC_is_MAYBE = isNFC_MAYBE($code_point)""" 4 |
| 444 |
.el .IP "\f(CW$NFC_is_MAYBE = isNFC_MAYBE($code_point)\fR" 4 |
| 445 |
.IX Item "$NFC_is_MAYBE = isNFC_MAYBE($code_point)" |
| 446 |
It returns a boolean of the derived property NFC_MAYBE |
| 447 |
(NFC_Quick_Check=Maybe). |
| 448 |
.ie n .IP """$NFKD_is_NO = isNFKD_NO($code_point)""" 4 |
| 449 |
.el .IP "\f(CW$NFKD_is_NO = isNFKD_NO($code_point)\fR" 4 |
| 450 |
.IX Item "$NFKD_is_NO = isNFKD_NO($code_point)" |
| 451 |
It returns a boolean of the derived property NFKD_NO |
| 452 |
(NFKD_Quick_Check=No). |
| 453 |
.ie n .IP """$NFKC_is_NO = isNFKC_NO($code_point)""" 4 |
| 454 |
.el .IP "\f(CW$NFKC_is_NO = isNFKC_NO($code_point)\fR" 4 |
| 455 |
.IX Item "$NFKC_is_NO = isNFKC_NO($code_point)" |
| 456 |
It returns a boolean of the derived property NFKC_NO |
| 457 |
(NFKC_Quick_Check=No). |
| 458 |
.ie n .IP """$NFKC_is_MAYBE = isNFKC_MAYBE($code_point)""" 4 |
| 459 |
.el .IP "\f(CW$NFKC_is_MAYBE = isNFKC_MAYBE($code_point)\fR" 4 |
| 460 |
.IX Item "$NFKC_is_MAYBE = isNFKC_MAYBE($code_point)" |
| 461 |
It returns a boolean of the derived property NFKC_MAYBE |
| 462 |
(NFKC_Quick_Check=Maybe). |
| 463 |
.SH EXPORT |
| 464 |
.IX Header "EXPORT" |
| 465 |
\&\f(CW\*(C`NFC\*(C'\fR, \f(CW\*(C`NFD\*(C'\fR, \f(CW\*(C`NFKC\*(C'\fR, \f(CW\*(C`NFKD\*(C'\fR: by default. |
| 466 |
.PP |
| 467 |
\&\f(CW\*(C`normalize\*(C'\fR and other some functions: on request. |
| 468 |
.SH CAVEATS |
| 469 |
.IX Header "CAVEATS" |
| 470 |
.IP "Perl's version vs. Unicode version" 4 |
| 471 |
.IX Item "Perl's version vs. Unicode version" |
| 472 |
Since this module refers to perl core's Unicode database in the directory |
| 473 |
\&\fI/lib/unicore\fR (or formerly \fI/lib/unicode\fR), the Unicode version of |
| 474 |
normalization implemented by this module depends on what has been |
| 475 |
compiled into your perl. The following table lists the default Unicode |
| 476 |
version that comes with various perl versions. (It is possible to change |
| 477 |
the Unicode version in any perl version to be any earlier Unicode version, |
| 478 |
so one could cause Unicode 3.2 to be used in any perl version starting with |
| 479 |
5.8.0. Read \fR\f(CI$Config{privlib}\fR\fI/unicore/README.perl\fR for details. |
| 480 |
.Sp |
| 481 |
.Vb 10 |
| 482 |
\& perl\*(Aqs version implemented Unicode version |
| 483 |
\& 5.6.1 3.0.1 |
| 484 |
\& 5.7.2 3.1.0 |
| 485 |
\& 5.7.3 3.1.1 (normalization is same as 3.1.0) |
| 486 |
\& 5.8.0 3.2.0 |
| 487 |
\& 5.8.1\-5.8.3 4.0.0 |
| 488 |
\& 5.8.4\-5.8.6 4.0.1 (normalization is same as 4.0.0) |
| 489 |
\& 5.8.7\-5.8.8 4.1.0 |
| 490 |
\& 5.10.0 5.0.0 |
| 491 |
\& 5.8.9, 5.10.1 5.1.0 |
| 492 |
\& 5.12.x 5.2.0 |
| 493 |
\& 5.14.x 6.0.0 |
| 494 |
\& 5.16.x 6.1.0 |
| 495 |
\& 5.18.x 6.2.0 |
| 496 |
\& 5.20.x 6.3.0 |
| 497 |
\& 5.22.x 7.0.0 |
| 498 |
.Ve |
| 499 |
.IP "Correction of decomposition mapping" 4 |
| 500 |
.IX Item "Correction of decomposition mapping" |
| 501 |
In older Unicode versions, a small number of characters (all of which are |
| 502 |
CJK compatibility ideographs as far as they have been found) may have |
| 503 |
an erroneous decomposition mapping (see |
| 504 |
\&\fIlib/unicore/NormalizationCorrections.txt\fR). |
| 505 |
Anyhow, this module will neither refer to |
| 506 |
\&\fIlib/unicore/NormalizationCorrections.txt\fR |
| 507 |
nor provide any specific version of normalization. Therefore this module |
| 508 |
running on an older perl with an older Unicode database may use |
| 509 |
the erroneous decomposition mapping blindly conforming to the Unicode database. |
| 510 |
.IP "Revised definition of canonical composition" 4 |
| 511 |
.IX Item "Revised definition of canonical composition" |
| 512 |
In Unicode 4.1.0, the definition D2 of canonical composition (which |
| 513 |
affects NFC and NFKC) has been changed (see Public Review Issue #29 |
| 514 |
and recent UAX #15). This module has used the newer definition |
| 515 |
since the version 0.07 (Oct 31, 2001). |
| 516 |
This module will not support the normalization according to the older |
| 517 |
definition, even if the Unicode version implemented by perl is |
| 518 |
lower than 4.1.0. |
| 519 |
.SH AUTHOR |
| 520 |
.IX Header "AUTHOR" |
| 521 |
SADAHIRO Tomoyuki <[email protected]> |
| 522 |
.PP |
| 523 |
Currently maintained by <perl5\[email protected]> |
| 524 |
.PP |
| 525 |
Copyright(C) 2001\-2012, SADAHIRO Tomoyuki. Japan. All rights reserved. |
| 526 |
.SH LICENSE |
| 527 |
.IX Header "LICENSE" |
| 528 |
This module is free software; you can redistribute it |
| 529 |
and/or modify it under the same terms as Perl itself. |
| 530 |
.SH "SEE ALSO" |
| 531 |
.IX Header "SEE ALSO" |
| 532 |
.IP <http://www.unicode.org/reports/tr15/> 4 |
| 533 |
.IX Item "<http://www.unicode.org/reports/tr15/>" |
| 534 |
Unicode Normalization Forms \- UAX #15 |
| 535 |
.IP <http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt> 4 |
| 536 |
.IX Item "<http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt>" |
| 537 |
Composition Exclusion Table |
| 538 |
.IP <http://www.unicode.org/Public/UNIDATA/DerivedNormalizationProps.txt> 4 |
| 539 |
.IX Item "<http://www.unicode.org/Public/UNIDATA/DerivedNormalizationProps.txt>" |
| 540 |
Derived Normalization Properties |
| 541 |
.IP <http://www.unicode.org/Public/UNIDATA/NormalizationCorrections.txt> 4 |
| 542 |
.IX Item "<http://www.unicode.org/Public/UNIDATA/NormalizationCorrections.txt>" |
| 543 |
Normalization Corrections |
| 544 |
.IP <http://www.unicode.org/review/pr\-29.html> 4 |
| 545 |
.IX Item "<http://www.unicode.org/review/pr-29.html>" |
| 546 |
Public Review Issue #29: Normalization Issue |
| 547 |
.IP <http://www.unicode.org/notes/tn5/> 4 |
| 548 |
.IX Item "<http://www.unicode.org/notes/tn5/>" |
| 549 |
Canonical Equivalence in Applications \- UTN #5 |