| 1 |
.\" -*- mode: troff; coding: utf-8 -*- |
| 2 |
.\" Automatically generated by Pod::Man 5.0102 (Pod::Simple 3.45) |
| 3 |
.\" |
| 4 |
.\" Standard preamble: |
| 5 |
.\" ======================================================================== |
| 6 |
.de Sp \" Vertical space (when we can't use .PP) |
| 7 |
.if t .sp .5v |
| 8 |
.if n .sp |
| 9 |
.. |
| 10 |
.de Vb \" Begin verbatim text |
| 11 |
.ft CW |
| 12 |
.nf |
| 13 |
.ne \\$1 |
| 14 |
.. |
| 15 |
.de Ve \" End verbatim text |
| 16 |
.ft R |
| 17 |
.fi |
| 18 |
.. |
| 19 |
.\" \*(C` and \*(C' are quotes in nroff, nothing in troff, for use with C<>. |
| 20 |
.ie n \{\ |
| 21 |
. ds C` "" |
| 22 |
. ds C' "" |
| 23 |
'br\} |
| 24 |
.el\{\ |
| 25 |
. ds C` |
| 26 |
. ds C' |
| 27 |
'br\} |
| 28 |
.\" |
| 29 |
.\" Escape single quotes in literal strings from groff's Unicode transform. |
| 30 |
.ie \n(.g .ds Aq \(aq |
| 31 |
.el .ds Aq ' |
| 32 |
.\" |
| 33 |
.\" If the F register is >0, we'll generate index entries on stderr for |
| 34 |
.\" titles (.TH), headers (.SH), subsections (.SS), items (.Ip), and index |
| 35 |
.\" entries marked with X<> in POD. Of course, you'll have to process the |
| 36 |
.\" output yourself in some meaningful fashion. |
| 37 |
.\" |
| 38 |
.\" Avoid warning from groff about undefined register 'F'. |
| 39 |
.de IX |
| 40 |
.. |
| 41 |
.nr rF 0 |
| 42 |
.if \n(.g .if rF .nr rF 1 |
| 43 |
.if (\n(rF:(\n(.g==0)) \{\ |
| 44 |
. if \nF \{\ |
| 45 |
. de IX |
| 46 |
. tm Index:\\$1\t\\n%\t"\\$2" |
| 47 |
.. |
| 48 |
. if !\nF==2 \{\ |
| 49 |
. nr % 0 |
| 50 |
. nr F 2 |
| 51 |
. \} |
| 52 |
. \} |
| 53 |
.\} |
| 54 |
.rr rF |
| 55 |
.\" ======================================================================== |
| 56 |
.\" |
| 57 |
.IX Title "PERLUNICOOK 1" |
| 58 |
.TH PERLUNICOOK 1 2025-11-16 "perl v5.40.1" "Perl Programmers Reference Guide" |
| 59 |
.\" For nroff, turn off justification. Always turn off hyphenation; it makes |
| 60 |
.\" way too many mistakes in technical documents. |
| 61 |
.if n .ad l |
| 62 |
.nh |
| 63 |
.SH NAME |
| 64 |
perlunicook \- cookbookish examples of handling Unicode in Perl |
| 65 |
.SH DESCRIPTION |
| 66 |
.IX Header "DESCRIPTION" |
| 67 |
This manpage contains short recipes demonstrating how to handle common Unicode |
| 68 |
operations in Perl, plus one complete program at the end. Any undeclared |
| 69 |
variables in individual recipes are assumed to have a previous appropriate |
| 70 |
value in them. |
| 71 |
.SH EXAMPLES |
| 72 |
.IX Header "EXAMPLES" |
| 73 |
.SS "℞ 0: Standard preamble" |
| 74 |
.IX Subsection "℞ 0: Standard preamble" |
| 75 |
Unless otherwise notes, all examples below require this standard preamble |
| 76 |
to work correctly, with the \f(CW\*(C`#!\*(C'\fR adjusted to work on your system: |
| 77 |
.PP |
| 78 |
.Vb 1 |
| 79 |
\& #!/usr/bin/env perl |
| 80 |
\& |
| 81 |
\& use v5.36; # or later to get "unicode_strings" feature, |
| 82 |
\& # plus strict, warnings |
| 83 |
\& use utf8; # so literals and identifiers can be in UTF\-8 |
| 84 |
\& use warnings qw(FATAL utf8); # fatalize encoding glitches |
| 85 |
\& use open qw(:std :encoding(UTF\-8)); # undeclared streams in UTF\-8 |
| 86 |
\& use charnames qw(:full :short); # unneeded in v5.16 |
| 87 |
.Ve |
| 88 |
.PP |
| 89 |
This \fIdoes\fR make even Unix programmers \f(CW\*(C`binmode\*(C'\fR your binary streams, |
| 90 |
or open them with \f(CW\*(C`:raw\*(C'\fR, but that's the only way to get at them |
| 91 |
portably anyway. |
| 92 |
.PP |
| 93 |
\&\fBWARNING\fR: \f(CW\*(C`use autodie\*(C'\fR (pre 2.26) and \f(CW\*(C`use open\*(C'\fR do not get along with each |
| 94 |
other. |
| 95 |
.SS "℞ 1: Generic Unicode-savvy filter" |
| 96 |
.IX Subsection "℞ 1: Generic Unicode-savvy filter" |
| 97 |
Always decompose on the way in, then recompose on the way out. |
| 98 |
.PP |
| 99 |
.Vb 1 |
| 100 |
\& use Unicode::Normalize; |
| 101 |
\& |
| 102 |
\& while (<>) { |
| 103 |
\& $_ = NFD($_); # decompose + reorder canonically |
| 104 |
\& ... |
| 105 |
\& } continue { |
| 106 |
\& print NFC($_); # recompose (where possible) + reorder canonically |
| 107 |
\& } |
| 108 |
.Ve |
| 109 |
.SS "℞ 2: Fine-tuning Unicode warnings" |
| 110 |
.IX Subsection "℞ 2: Fine-tuning Unicode warnings" |
| 111 |
As of v5.14, Perl distinguishes three subclasses of UTF‑8 warnings. |
| 112 |
.PP |
| 113 |
.Vb 4 |
| 114 |
\& use v5.14; # subwarnings unavailable any earlier |
| 115 |
\& no warnings "nonchar"; # the 66 forbidden non\-characters |
| 116 |
\& no warnings "surrogate"; # UTF\-16/CESU\-8 nonsense |
| 117 |
\& no warnings "non_unicode"; # for codepoints over 0x10_FFFF |
| 118 |
.Ve |
| 119 |
.SS "℞ 3: Declare source in utf8 for identifiers and literals" |
| 120 |
.IX Subsection "℞ 3: Declare source in utf8 for identifiers and literals" |
| 121 |
Without the all-critical \f(CW\*(C`use utf8\*(C'\fR declaration, putting UTF‑8 in your |
| 122 |
literals and identifiers won’t work right. If you used the standard |
| 123 |
preamble just given above, this already happened. If you did, you can |
| 124 |
do things like this: |
| 125 |
.PP |
| 126 |
.Vb 1 |
| 127 |
\& use utf8; |
| 128 |
\& |
| 129 |
\& my $measure = "Ångström"; |
| 130 |
\& my @μsoft = qw( cp852 cp1251 cp1252 ); |
| 131 |
\& my @ὑπέρμεγας = qw( ὑπέρ μεγας ); |
| 132 |
\& my @鯉 = qw( koi8\-f koi8\-u koi8\-r ); |
| 133 |
\& my $motto = "👪 💗 🐪"; # FAMILY, GROWING HEART, DROMEDARY CAMEL |
| 134 |
.Ve |
| 135 |
.PP |
| 136 |
If you forget \f(CW\*(C`use utf8\*(C'\fR, high bytes will be misunderstood as |
| 137 |
separate characters, and nothing will work right. |
| 138 |
.SS "℞ 4: Characters and their numbers" |
| 139 |
.IX Subsection "℞ 4: Characters and their numbers" |
| 140 |
The \f(CW\*(C`ord\*(C'\fR and \f(CW\*(C`chr\*(C'\fR functions work transparently on all codepoints, |
| 141 |
not just on ASCII alone — nor in fact, not even just on Unicode alone. |
| 142 |
.PP |
| 143 |
.Vb 3 |
| 144 |
\& # ASCII characters |
| 145 |
\& ord("A") |
| 146 |
\& chr(65) |
| 147 |
\& |
| 148 |
\& # characters from the Basic Multilingual Plane |
| 149 |
\& ord("Σ") |
| 150 |
\& chr(0x3A3) |
| 151 |
\& |
| 152 |
\& # beyond the BMP |
| 153 |
\& ord("𝑛") # MATHEMATICAL ITALIC SMALL N |
| 154 |
\& chr(0x1D45B) |
| 155 |
\& |
| 156 |
\& # beyond Unicode! (up to MAXINT) |
| 157 |
\& ord("\ex{20_0000}") |
| 158 |
\& chr(0x20_0000) |
| 159 |
.Ve |
| 160 |
.SS "℞ 5: Unicode literals by character number" |
| 161 |
.IX Subsection "℞ 5: Unicode literals by character number" |
| 162 |
In an interpolated literal, whether a double-quoted string or a |
| 163 |
regex, you may specify a character by its number using the |
| 164 |
\&\f(CW\*(C`\ex{\fR\f(CIHHHHHH\fR\f(CW}\*(C'\fR escape. |
| 165 |
.PP |
| 166 |
.Vb 2 |
| 167 |
\& String: "\ex{3a3}" |
| 168 |
\& Regex: /\ex{3a3}/ |
| 169 |
\& |
| 170 |
\& String: "\ex{1d45b}" |
| 171 |
\& Regex: /\ex{1d45b}/ |
| 172 |
\& |
| 173 |
\& # even non\-BMP ranges in regex work fine |
| 174 |
\& /[\ex{1D434}\-\ex{1D467}]/ |
| 175 |
.Ve |
| 176 |
.SS "℞ 6: Get character name by number" |
| 177 |
.IX Subsection "℞ 6: Get character name by number" |
| 178 |
.Vb 2 |
| 179 |
\& use charnames (); |
| 180 |
\& my $name = charnames::viacode(0x03A3); |
| 181 |
.Ve |
| 182 |
.SS "℞ 7: Get character number by name" |
| 183 |
.IX Subsection "℞ 7: Get character number by name" |
| 184 |
.Vb 2 |
| 185 |
\& use charnames (); |
| 186 |
\& my $number = charnames::vianame("GREEK CAPITAL LETTER SIGMA"); |
| 187 |
.Ve |
| 188 |
.SS "℞ 8: Unicode named characters" |
| 189 |
.IX Subsection "℞ 8: Unicode named characters" |
| 190 |
Use the \f(CW\*(C`\eN{\fR\f(CIcharname\fR\f(CW}\*(C'\fR notation to get the character |
| 191 |
by that name for use in interpolated literals (double-quoted |
| 192 |
strings and regexes). In v5.16, there is an implicit |
| 193 |
.PP |
| 194 |
.Vb 1 |
| 195 |
\& use charnames qw(:full :short); |
| 196 |
.Ve |
| 197 |
.PP |
| 198 |
But prior to v5.16, you must be explicit about which set of charnames you |
| 199 |
want. The \f(CW\*(C`:full\*(C'\fR names are the official Unicode character name, alias, or |
| 200 |
sequence, which all share a namespace. |
| 201 |
.PP |
| 202 |
.Vb 1 |
| 203 |
\& use charnames qw(:full :short latin greek); |
| 204 |
\& |
| 205 |
\& "\eN{MATHEMATICAL ITALIC SMALL N}" # :full |
| 206 |
\& "\eN{GREEK CAPITAL LETTER SIGMA}" # :full |
| 207 |
.Ve |
| 208 |
.PP |
| 209 |
Anything else is a Perl-specific convenience abbreviation. Specify one or |
| 210 |
more scripts by names if you want short names that are script-specific. |
| 211 |
.PP |
| 212 |
.Vb 3 |
| 213 |
\& "\eN{Greek:Sigma}" # :short |
| 214 |
\& "\eN{ae}" # latin |
| 215 |
\& "\eN{epsilon}" # greek |
| 216 |
.Ve |
| 217 |
.PP |
| 218 |
The v5.16 release also supports a \f(CW\*(C`:loose\*(C'\fR import for loose matching of |
| 219 |
character names, which works just like loose matching of property names: |
| 220 |
that is, it disregards case, whitespace, and underscores: |
| 221 |
.PP |
| 222 |
.Vb 1 |
| 223 |
\& "\eN{euro sign}" # :loose (from v5.16) |
| 224 |
.Ve |
| 225 |
.PP |
| 226 |
Starting in v5.32, you can also use |
| 227 |
.PP |
| 228 |
.Vb 1 |
| 229 |
\& qr/\ep{name=euro sign}/ |
| 230 |
.Ve |
| 231 |
.PP |
| 232 |
to get official Unicode named characters in regular expressions. Loose |
| 233 |
matching is always done for these. |
| 234 |
.SS "℞ 9: Unicode named sequences" |
| 235 |
.IX Subsection "℞ 9: Unicode named sequences" |
| 236 |
These look just like character names but return multiple codepoints. |
| 237 |
Notice the \f(CW%vx\fR vector-print functionality in \f(CW\*(C`printf\*(C'\fR. |
| 238 |
.PP |
| 239 |
.Vb 4 |
| 240 |
\& use charnames qw(:full); |
| 241 |
\& my $seq = "\eN{LATIN CAPITAL LETTER A WITH MACRON AND GRAVE}"; |
| 242 |
\& printf "U+%v04X\en", $seq; |
| 243 |
\& U+0100.0300 |
| 244 |
.Ve |
| 245 |
.SS "℞ 10: Custom named characters" |
| 246 |
.IX Subsection "℞ 10: Custom named characters" |
| 247 |
Use \f(CW\*(C`:alias\*(C'\fR to give your own lexically scoped nicknames to existing |
| 248 |
characters, or even to give unnamed private-use characters useful names. |
| 249 |
.PP |
| 250 |
.Vb 4 |
| 251 |
\& use charnames ":full", ":alias" => { |
| 252 |
\& ecute => "LATIN SMALL LETTER E WITH ACUTE", |
| 253 |
\& "APPLE LOGO" => 0xF8FF, # private use character |
| 254 |
\& }; |
| 255 |
\& |
| 256 |
\& "\eN{ecute}" |
| 257 |
\& "\eN{APPLE LOGO}" |
| 258 |
.Ve |
| 259 |
.SS "℞ 11: Names of CJK codepoints" |
| 260 |
.IX Subsection "℞ 11: Names of CJK codepoints" |
| 261 |
Sinograms like “東京” come back with character names of |
| 262 |
\&\f(CW\*(C`CJK UNIFIED IDEOGRAPH\-6771\*(C'\fR and \f(CW\*(C`CJK UNIFIED IDEOGRAPH\-4EAC\*(C'\fR, |
| 263 |
because their “names” vary. The CPAN \f(CW\*(C`Unicode::Unihan\*(C'\fR module |
| 264 |
has a large database for decoding these (and a whole lot more), provided you |
| 265 |
know how to understand its output. |
| 266 |
.PP |
| 267 |
.Vb 8 |
| 268 |
\& # cpan \-i Unicode::Unihan |
| 269 |
\& use Unicode::Unihan; |
| 270 |
\& my $str = "東京"; |
| 271 |
\& my $unhan = Unicode::Unihan\->new; |
| 272 |
\& for my $lang (qw(Mandarin Cantonese Korean JapaneseOn JapaneseKun)) { |
| 273 |
\& printf "CJK $str in %\-12s is ", $lang; |
| 274 |
\& say $unhan\->$lang($str); |
| 275 |
\& } |
| 276 |
.Ve |
| 277 |
.PP |
| 278 |
prints: |
| 279 |
.PP |
| 280 |
.Vb 5 |
| 281 |
\& CJK 東京 in Mandarin is DONG1JING1 |
| 282 |
\& CJK 東京 in Cantonese is dung1ging1 |
| 283 |
\& CJK 東京 in Korean is TONGKYENG |
| 284 |
\& CJK 東京 in JapaneseOn is TOUKYOU KEI KIN |
| 285 |
\& CJK 東京 in JapaneseKun is HIGASHI AZUMAMIYAKO |
| 286 |
.Ve |
| 287 |
.PP |
| 288 |
If you have a specific romanization scheme in mind, |
| 289 |
use the specific module: |
| 290 |
.PP |
| 291 |
.Vb 5 |
| 292 |
\& # cpan \-i Lingua::JA::Romanize::Japanese |
| 293 |
\& use Lingua::JA::Romanize::Japanese; |
| 294 |
\& my $k2r = Lingua::JA::Romanize::Japanese\->new; |
| 295 |
\& my $str = "東京"; |
| 296 |
\& say "Japanese for $str is ", $k2r\->chars($str); |
| 297 |
.Ve |
| 298 |
.PP |
| 299 |
prints |
| 300 |
.PP |
| 301 |
.Vb 1 |
| 302 |
\& Japanese for 東京 is toukyou |
| 303 |
.Ve |
| 304 |
.SS "℞ 12: Explicit encode/decode" |
| 305 |
.IX Subsection "℞ 12: Explicit encode/decode" |
| 306 |
On rare occasion, such as a database read, you may be |
| 307 |
given encoded text you need to decode. |
| 308 |
.PP |
| 309 |
.Vb 1 |
| 310 |
\& use Encode qw(encode decode); |
| 311 |
\& |
| 312 |
\& my $chars = decode("shiftjis", $bytes, 1); |
| 313 |
\& # OR |
| 314 |
\& my $bytes = encode("MIME\-Header\-ISO_2022_JP", $chars, 1); |
| 315 |
.Ve |
| 316 |
.PP |
| 317 |
For streams all in the same encoding, don't use encode/decode; instead |
| 318 |
set the file encoding when you open the file or immediately after with |
| 319 |
\&\f(CW\*(C`binmode\*(C'\fR as described later below. |
| 320 |
.SS "℞ 13: Decode program arguments as utf8" |
| 321 |
.IX Subsection "℞ 13: Decode program arguments as utf8" |
| 322 |
.Vb 6 |
| 323 |
\& $ perl \-CA ... |
| 324 |
\& or |
| 325 |
\& $ export PERL_UNICODE=A |
| 326 |
\& or |
| 327 |
\& use Encode qw(decode); |
| 328 |
\& @ARGV = map { decode(\*(AqUTF\-8\*(Aq, $_, 1) } @ARGV; |
| 329 |
.Ve |
| 330 |
.SS "℞ 14: Decode program arguments as locale encoding" |
| 331 |
.IX Subsection "℞ 14: Decode program arguments as locale encoding" |
| 332 |
.Vb 3 |
| 333 |
\& # cpan \-i Encode::Locale |
| 334 |
\& use Encode qw(locale); |
| 335 |
\& use Encode::Locale; |
| 336 |
\& |
| 337 |
\& # use "locale" as an arg to encode/decode |
| 338 |
\& @ARGV = map { decode(locale => $_, 1) } @ARGV; |
| 339 |
.Ve |
| 340 |
.SS "℞ 15: Declare STD{IN,OUT,ERR} to be utf8" |
| 341 |
.IX Subsection "℞ 15: Declare STD{IN,OUT,ERR} to be utf8" |
| 342 |
Use a command-line option, an environment variable, or else |
| 343 |
call \f(CW\*(C`binmode\*(C'\fR explicitly: |
| 344 |
.PP |
| 345 |
.Vb 9 |
| 346 |
\& $ perl \-CS ... |
| 347 |
\& or |
| 348 |
\& $ export PERL_UNICODE=S |
| 349 |
\& or |
| 350 |
\& use open qw(:std :encoding(UTF\-8)); |
| 351 |
\& or |
| 352 |
\& binmode(STDIN, ":encoding(UTF\-8)"); |
| 353 |
\& binmode(STDOUT, ":utf8"); |
| 354 |
\& binmode(STDERR, ":utf8"); |
| 355 |
.Ve |
| 356 |
.SS "℞ 16: Declare STD{IN,OUT,ERR} to be in locale encoding" |
| 357 |
.IX Subsection "℞ 16: Declare STD{IN,OUT,ERR} to be in locale encoding" |
| 358 |
.Vb 3 |
| 359 |
\& # cpan \-i Encode::Locale |
| 360 |
\& use Encode; |
| 361 |
\& use Encode::Locale; |
| 362 |
\& |
| 363 |
\& # or as a stream for binmode or open |
| 364 |
\& binmode STDIN, ":encoding(console_in)" if \-t STDIN; |
| 365 |
\& binmode STDOUT, ":encoding(console_out)" if \-t STDOUT; |
| 366 |
\& binmode STDERR, ":encoding(console_out)" if \-t STDERR; |
| 367 |
.Ve |
| 368 |
.SS "℞ 17: Make file I/O default to utf8" |
| 369 |
.IX Subsection "℞ 17: Make file I/O default to utf8" |
| 370 |
Files opened without an encoding argument will be in UTF\-8: |
| 371 |
.PP |
| 372 |
.Vb 5 |
| 373 |
\& $ perl \-CD ... |
| 374 |
\& or |
| 375 |
\& $ export PERL_UNICODE=D |
| 376 |
\& or |
| 377 |
\& use open qw(:encoding(UTF\-8)); |
| 378 |
.Ve |
| 379 |
.SS "℞ 18: Make all I/O and args default to utf8" |
| 380 |
.IX Subsection "℞ 18: Make all I/O and args default to utf8" |
| 381 |
.Vb 7 |
| 382 |
\& $ perl \-CSDA ... |
| 383 |
\& or |
| 384 |
\& $ export PERL_UNICODE=SDA |
| 385 |
\& or |
| 386 |
\& use open qw(:std :encoding(UTF\-8)); |
| 387 |
\& use Encode qw(decode); |
| 388 |
\& @ARGV = map { decode(\*(AqUTF\-8\*(Aq, $_, 1) } @ARGV; |
| 389 |
.Ve |
| 390 |
.SS "℞ 19: Open file with specific encoding" |
| 391 |
.IX Subsection "℞ 19: Open file with specific encoding" |
| 392 |
Specify stream encoding. This is the normal way |
| 393 |
to deal with encoded text, not by calling low-level |
| 394 |
functions. |
| 395 |
.PP |
| 396 |
.Vb 7 |
| 397 |
\& # input file |
| 398 |
\& open(my $in_file, "< :encoding(UTF\-16)", "wintext"); |
| 399 |
\& OR |
| 400 |
\& open(my $in_file, "<", "wintext"); |
| 401 |
\& binmode($in_file, ":encoding(UTF\-16)"); |
| 402 |
\& THEN |
| 403 |
\& my $line = <$in_file>; |
| 404 |
\& |
| 405 |
\& # output file |
| 406 |
\& open($out_file, "> :encoding(cp1252)", "wintext"); |
| 407 |
\& OR |
| 408 |
\& open(my $out_file, ">", "wintext"); |
| 409 |
\& binmode($out_file, ":encoding(cp1252)"); |
| 410 |
\& THEN |
| 411 |
\& print $out_file "some text\en"; |
| 412 |
.Ve |
| 413 |
.PP |
| 414 |
More layers than just the encoding can be specified here. For example, |
| 415 |
the incantation \f(CW":raw :encoding(UTF\-16LE) :crlf"\fR includes implicit |
| 416 |
CRLF handling. |
| 417 |
.SS "℞ 20: Unicode casing" |
| 418 |
.IX Subsection "℞ 20: Unicode casing" |
| 419 |
Unicode casing is very different from ASCII casing. |
| 420 |
.PP |
| 421 |
.Vb 2 |
| 422 |
\& uc("henry ⅷ") # "HENRY Ⅷ" |
| 423 |
\& uc("tschüß") # "TSCHÜSS" notice ß => SS |
| 424 |
\& |
| 425 |
\& # both are true: |
| 426 |
\& "tschüß" =~ /TSCHÜSS/i # notice ß => SS |
| 427 |
\& "Σίσυφος" =~ /ΣΊΣΥΦΟΣ/i # notice Σ,σ,ς sameness |
| 428 |
.Ve |
| 429 |
.SS "℞ 21: Unicode case-insensitive comparisons" |
| 430 |
.IX Subsection "℞ 21: Unicode case-insensitive comparisons" |
| 431 |
Also available in the CPAN Unicode::CaseFold module, |
| 432 |
the new \f(CW\*(C`fc\*(C'\fR “foldcase” function from v5.16 grants |
| 433 |
access to the same Unicode casefolding as the \f(CW\*(C`/i\*(C'\fR |
| 434 |
pattern modifier has always used: |
| 435 |
.PP |
| 436 |
.Vb 1 |
| 437 |
\& use feature "fc"; # fc() function is from v5.16 |
| 438 |
\& |
| 439 |
\& # sort case\-insensitively |
| 440 |
\& my @sorted = sort { fc($a) cmp fc($b) } @list; |
| 441 |
\& |
| 442 |
\& # both are true: |
| 443 |
\& fc("tschüß") eq fc("TSCHÜSS") |
| 444 |
\& fc("Σίσυφος") eq fc("ΣΊΣΥΦΟΣ") |
| 445 |
.Ve |
| 446 |
.SS "℞ 22: Match Unicode linebreak sequence in regex" |
| 447 |
.IX Subsection "℞ 22: Match Unicode linebreak sequence in regex" |
| 448 |
A Unicode linebreak matches the two-character CRLF |
| 449 |
grapheme or any of seven vertical whitespace characters. |
| 450 |
Good for dealing with textfiles coming from different |
| 451 |
operating systems. |
| 452 |
.PP |
| 453 |
.Vb 1 |
| 454 |
\& \eR |
| 455 |
\& |
| 456 |
\& s/\eR/\en/g; # normalize all linebreaks to \en |
| 457 |
.Ve |
| 458 |
.SS "℞ 23: Get character category" |
| 459 |
.IX Subsection "℞ 23: Get character category" |
| 460 |
Find the general category of a numeric codepoint. |
| 461 |
.PP |
| 462 |
.Vb 2 |
| 463 |
\& use Unicode::UCD qw(charinfo); |
| 464 |
\& my $cat = charinfo(0x3A3)\->{category}; # "Lu" |
| 465 |
.Ve |
| 466 |
.SS "℞ 24: Disabling Unicode-awareness in builtin charclasses" |
| 467 |
.IX Subsection "℞ 24: Disabling Unicode-awareness in builtin charclasses" |
| 468 |
Disable \f(CW\*(C`\ew\*(C'\fR, \f(CW\*(C`\eb\*(C'\fR, \f(CW\*(C`\es\*(C'\fR, \f(CW\*(C`\ed\*(C'\fR, and the POSIX |
| 469 |
classes from working correctly on Unicode either in this |
| 470 |
scope, or in just one regex. |
| 471 |
.PP |
| 472 |
.Vb 2 |
| 473 |
\& use v5.14; |
| 474 |
\& use re "/a"; |
| 475 |
\& |
| 476 |
\& # OR |
| 477 |
\& |
| 478 |
\& my($num) = $str =~ /(\ed+)/a; |
| 479 |
.Ve |
| 480 |
.PP |
| 481 |
Or use specific un-Unicode properties, like \f(CW\*(C`\ep{ahex}\*(C'\fR |
| 482 |
and \f(CW\*(C`\ep{POSIX_Digit\*(C'\fR}. Properties still work normally |
| 483 |
no matter what charset modifiers (\f(CW\*(C`/d /u /l /a /aa\*(C'\fR) |
| 484 |
should be effect. |
| 485 |
.SS "℞ 25: Match Unicode properties in regex with \ep, \eP" |
| 486 |
.IX Subsection "℞ 25: Match Unicode properties in regex with p, P" |
| 487 |
These all match a single codepoint with the given |
| 488 |
property. Use \f(CW\*(C`\eP\*(C'\fR in place of \f(CW\*(C`\ep\*(C'\fR to match |
| 489 |
one codepoint lacking that property. |
| 490 |
.PP |
| 491 |
.Vb 8 |
| 492 |
\& \epL, \epN, \epS, \epP, \epM, \epZ, \epC |
| 493 |
\& \ep{Sk}, \ep{Ps}, \ep{Lt} |
| 494 |
\& \ep{alpha}, \ep{upper}, \ep{lower} |
| 495 |
\& \ep{Latin}, \ep{Greek} |
| 496 |
\& \ep{script_extensions=Latin}, \ep{scx=Greek} |
| 497 |
\& \ep{East_Asian_Width=Wide}, \ep{EA=W} |
| 498 |
\& \ep{Line_Break=Hyphen}, \ep{LB=HY} |
| 499 |
\& \ep{Numeric_Value=4}, \ep{NV=4} |
| 500 |
.Ve |
| 501 |
.SS "℞ 26: Custom character properties" |
| 502 |
.IX Subsection "℞ 26: Custom character properties" |
| 503 |
Define at compile-time your own custom character |
| 504 |
properties for use in regexes. |
| 505 |
.PP |
| 506 |
.Vb 2 |
| 507 |
\& # using private\-use characters |
| 508 |
\& sub In_Tengwar { "E000\etE07F\en" } |
| 509 |
\& |
| 510 |
\& if (/\ep{In_Tengwar}/) { ... } |
| 511 |
\& |
| 512 |
\& # blending existing properties |
| 513 |
\& sub Is_GraecoRoman_Title {<<\*(AqEND_OF_SET\*(Aq} |
| 514 |
\& +utf8::IsLatin |
| 515 |
\& +utf8::IsGreek |
| 516 |
\& &utf8::IsTitle |
| 517 |
\& END_OF_SET |
| 518 |
\& |
| 519 |
\& if (/\ep{Is_GraecoRoman_Title}/ { ... } |
| 520 |
.Ve |
| 521 |
.SS "℞ 27: Unicode normalization" |
| 522 |
.IX Subsection "℞ 27: Unicode normalization" |
| 523 |
Typically render into NFD on input and NFC on output. Using NFKC or NFKD |
| 524 |
functions improves recall on searches, assuming you've already done to the |
| 525 |
same text to be searched. Note that this is about much more than just pre\- |
| 526 |
combined compatibility glyphs; it also reorders marks according to their |
| 527 |
canonical combining classes and weeds out singletons. |
| 528 |
.PP |
| 529 |
.Vb 5 |
| 530 |
\& use Unicode::Normalize; |
| 531 |
\& my $nfd = NFD($orig); |
| 532 |
\& my $nfc = NFC($orig); |
| 533 |
\& my $nfkd = NFKD($orig); |
| 534 |
\& my $nfkc = NFKC($orig); |
| 535 |
.Ve |
| 536 |
.SS "℞ 28: Convert non-ASCII Unicode numerics" |
| 537 |
.IX Subsection "℞ 28: Convert non-ASCII Unicode numerics" |
| 538 |
Unless you’ve used \f(CW\*(C`/a\*(C'\fR or \f(CW\*(C`/aa\*(C'\fR, \f(CW\*(C`\ed\*(C'\fR matches more than |
| 539 |
ASCII digits only, but Perl’s implicit string-to-number |
| 540 |
conversion does not current recognize these. Here’s how to |
| 541 |
convert such strings manually. |
| 542 |
.PP |
| 543 |
.Vb 8 |
| 544 |
\& use v5.14; # needed for num() function |
| 545 |
\& use Unicode::UCD qw(num); |
| 546 |
\& my $str = "got Ⅻ and ४५६७ and ⅞ and here"; |
| 547 |
\& my @nums = (); |
| 548 |
\& while ($str =~ /(\ed+|\eN)/g) { # not just ASCII! |
| 549 |
\& push @nums, num($1); |
| 550 |
\& } |
| 551 |
\& say "@nums"; # 12 4567 0.875 |
| 552 |
\& |
| 553 |
\& use charnames qw(:full); |
| 554 |
\& my $nv = num("\eN{RUMI DIGIT ONE}\eN{RUMI DIGIT TWO}"); |
| 555 |
.Ve |
| 556 |
.SS "℞ 29: Match Unicode grapheme cluster in regex" |
| 557 |
.IX Subsection "℞ 29: Match Unicode grapheme cluster in regex" |
| 558 |
Programmer-visible “characters” are codepoints matched by \f(CW\*(C`/./s\*(C'\fR, |
| 559 |
but user-visible “characters” are graphemes matched by \f(CW\*(C`/\eX/\*(C'\fR. |
| 560 |
.PP |
| 561 |
.Vb 3 |
| 562 |
\& # Find vowel *plus* any combining diacritics,underlining,etc. |
| 563 |
\& my $nfd = NFD($orig); |
| 564 |
\& $nfd =~ / (?=[aeiou]) \eX /xi |
| 565 |
.Ve |
| 566 |
.SS "℞ 30: Extract by grapheme instead of by codepoint (regex)" |
| 567 |
.IX Subsection "℞ 30: Extract by grapheme instead of by codepoint (regex)" |
| 568 |
.Vb 2 |
| 569 |
\& # match and grab five first graphemes |
| 570 |
\& my($first_five) = $str =~ /^ ( \eX{5} ) /x; |
| 571 |
.Ve |
| 572 |
.SS "℞ 31: Extract by grapheme instead of by codepoint (substr)" |
| 573 |
.IX Subsection "℞ 31: Extract by grapheme instead of by codepoint (substr)" |
| 574 |
.Vb 4 |
| 575 |
\& # cpan \-i Unicode::GCString |
| 576 |
\& use Unicode::GCString; |
| 577 |
\& my $gcs = Unicode::GCString\->new($str); |
| 578 |
\& my $first_five = $gcs\->substr(0, 5); |
| 579 |
.Ve |
| 580 |
.SS "℞ 32: Reverse string by grapheme" |
| 581 |
.IX Subsection "℞ 32: Reverse string by grapheme" |
| 582 |
Reversing by codepoint messes up diacritics, mistakenly converting |
| 583 |
\&\f(CW\*(C`crème brûlée\*(C'\fR into \f(CW\*(C`éel̂urb em̀erc\*(C'\fR instead of into \f(CW\*(C`eélûrb emèrc\*(C'\fR; |
| 584 |
so reverse by grapheme instead. Both these approaches work |
| 585 |
right no matter what normalization the string is in: |
| 586 |
.PP |
| 587 |
.Vb 1 |
| 588 |
\& $str = join("", reverse $str =~ /\eX/g); |
| 589 |
\& |
| 590 |
\& # OR: cpan \-i Unicode::GCString |
| 591 |
\& use Unicode::GCString; |
| 592 |
\& $str = reverse Unicode::GCString\->new($str); |
| 593 |
.Ve |
| 594 |
.SS "℞ 33: String length in graphemes" |
| 595 |
.IX Subsection "℞ 33: String length in graphemes" |
| 596 |
The string \f(CW\*(C`brûlée\*(C'\fR has six graphemes but up to eight codepoints. |
| 597 |
This counts by grapheme, not by codepoint: |
| 598 |
.PP |
| 599 |
.Vb 3 |
| 600 |
\& my $str = "brûlée"; |
| 601 |
\& my $count = 0; |
| 602 |
\& while ($str =~ /\eX/g) { $count++ } |
| 603 |
\& |
| 604 |
\& # OR: cpan \-i Unicode::GCString |
| 605 |
\& use Unicode::GCString; |
| 606 |
\& my $gcs = Unicode::GCString\->new($str); |
| 607 |
\& my $count = $gcs\->length; |
| 608 |
.Ve |
| 609 |
.SS "℞ 34: Unicode column-width for printing" |
| 610 |
.IX Subsection "℞ 34: Unicode column-width for printing" |
| 611 |
Perl’s \f(CW\*(C`printf\*(C'\fR, \f(CW\*(C`sprintf\*(C'\fR, and \f(CW\*(C`format\*(C'\fR think all |
| 612 |
codepoints take up 1 print column, but many take 0 or 2. |
| 613 |
Here to show that normalization makes no difference, |
| 614 |
we print out both forms: |
| 615 |
.PP |
| 616 |
.Vb 2 |
| 617 |
\& use Unicode::GCString; |
| 618 |
\& use Unicode::Normalize; |
| 619 |
\& |
| 620 |
\& my @words = qw/crème brûlée/; |
| 621 |
\& @words = map { NFC($_), NFD($_) } @words; |
| 622 |
\& |
| 623 |
\& for my $str (@words) { |
| 624 |
\& my $gcs = Unicode::GCString\->new($str); |
| 625 |
\& my $cols = $gcs\->columns; |
| 626 |
\& my $pad = " " x (10 \- $cols); |
| 627 |
\& say str, $pad, " |"; |
| 628 |
\& } |
| 629 |
.Ve |
| 630 |
.PP |
| 631 |
generates this to show that it pads correctly no matter |
| 632 |
the normalization: |
| 633 |
.PP |
| 634 |
.Vb 4 |
| 635 |
\& crème | |
| 636 |
\& crème | |
| 637 |
\& brûlée | |
| 638 |
\& brûlée | |
| 639 |
.Ve |
| 640 |
.SS "℞ 35: Unicode collation" |
| 641 |
.IX Subsection "℞ 35: Unicode collation" |
| 642 |
Text sorted by numeric codepoint follows no reasonable alphabetic order; |
| 643 |
use the UCA for sorting text. |
| 644 |
.PP |
| 645 |
.Vb 3 |
| 646 |
\& use Unicode::Collate; |
| 647 |
\& my $col = Unicode::Collate\->new(); |
| 648 |
\& my @list = $col\->sort(@old_list); |
| 649 |
.Ve |
| 650 |
.PP |
| 651 |
See the \fIucsort\fR program from the Unicode::Tussle CPAN module |
| 652 |
for a convenient command-line interface to this module. |
| 653 |
.SS "℞ 36: Case\- \fIand\fP accent-insensitive Unicode sort" |
| 654 |
.IX Subsection "℞ 36: Case- and accent-insensitive Unicode sort" |
| 655 |
Specify a collation strength of level 1 to ignore case and |
| 656 |
diacritics, only looking at the basic character. |
| 657 |
.PP |
| 658 |
.Vb 3 |
| 659 |
\& use Unicode::Collate; |
| 660 |
\& my $col = Unicode::Collate\->new(level => 1); |
| 661 |
\& my @list = $col\->sort(@old_list); |
| 662 |
.Ve |
| 663 |
.SS "℞ 37: Unicode locale collation" |
| 664 |
.IX Subsection "℞ 37: Unicode locale collation" |
| 665 |
Some locales have special sorting rules. |
| 666 |
.PP |
| 667 |
.Vb 4 |
| 668 |
\& # either use v5.12, OR: cpan \-i Unicode::Collate::Locale |
| 669 |
\& use Unicode::Collate::Locale; |
| 670 |
\& my $col = Unicode::Collate::Locale\->new(locale => "de_\|_phonebook"); |
| 671 |
\& my @list = $col\->sort(@old_list); |
| 672 |
.Ve |
| 673 |
.PP |
| 674 |
The \fIucsort\fR program mentioned above accepts a \f(CW\*(C`\-\-locale\*(C'\fR parameter. |
| 675 |
.ie n .SS "℞ 38: Making ""cmp"" work on text instead of codepoints" |
| 676 |
.el .SS "℞ 38: Making \f(CWcmp\fP work on text instead of codepoints" |
| 677 |
.IX Subsection "℞ 38: Making cmp work on text instead of codepoints" |
| 678 |
Instead of this: |
| 679 |
.PP |
| 680 |
.Vb 5 |
| 681 |
\& @srecs = sort { |
| 682 |
\& $b\->{AGE} <=> $a\->{AGE} |
| 683 |
\& || |
| 684 |
\& $a\->{NAME} cmp $b\->{NAME} |
| 685 |
\& } @recs; |
| 686 |
.Ve |
| 687 |
.PP |
| 688 |
Use this: |
| 689 |
.PP |
| 690 |
.Vb 9 |
| 691 |
\& my $coll = Unicode::Collate\->new(); |
| 692 |
\& for my $rec (@recs) { |
| 693 |
\& $rec\->{NAME_key} = $coll\->getSortKey( $rec\->{NAME} ); |
| 694 |
\& } |
| 695 |
\& @srecs = sort { |
| 696 |
\& $b\->{AGE} <=> $a\->{AGE} |
| 697 |
\& || |
| 698 |
\& $a\->{NAME_key} cmp $b\->{NAME_key} |
| 699 |
\& } @recs; |
| 700 |
.Ve |
| 701 |
.SS "℞ 39: Case\- \fIand\fP accent-insensitive comparisons" |
| 702 |
.IX Subsection "℞ 39: Case- and accent-insensitive comparisons" |
| 703 |
Use a collator object to compare Unicode text by character |
| 704 |
instead of by codepoint. |
| 705 |
.PP |
| 706 |
.Vb 5 |
| 707 |
\& use Unicode::Collate; |
| 708 |
\& my $es = Unicode::Collate\->new( |
| 709 |
\& level => 1, |
| 710 |
\& normalization => undef |
| 711 |
\& ); |
| 712 |
\& |
| 713 |
\& # now both are true: |
| 714 |
\& $es\->eq("García", "GARCIA" ); |
| 715 |
\& $es\->eq("Márquez", "MARQUEZ"); |
| 716 |
.Ve |
| 717 |
.SS "℞ 40: Case\- \fIand\fP accent-insensitive locale comparisons" |
| 718 |
.IX Subsection "℞ 40: Case- and accent-insensitive locale comparisons" |
| 719 |
Same, but in a specific locale. |
| 720 |
.PP |
| 721 |
.Vb 3 |
| 722 |
\& my $de = Unicode::Collate::Locale\->new( |
| 723 |
\& locale => "de_\|_phonebook", |
| 724 |
\& ); |
| 725 |
\& |
| 726 |
\& # now this is true: |
| 727 |
\& $de\->eq("tschüß", "TSCHUESS"); # notice ü => UE, ß => SS |
| 728 |
.Ve |
| 729 |
.SS "℞ 41: Unicode linebreaking" |
| 730 |
.IX Subsection "℞ 41: Unicode linebreaking" |
| 731 |
Break up text into lines according to Unicode rules. |
| 732 |
.PP |
| 733 |
.Vb 3 |
| 734 |
\& # cpan \-i Unicode::LineBreak |
| 735 |
\& use Unicode::LineBreak; |
| 736 |
\& use charnames qw(:full); |
| 737 |
\& |
| 738 |
\& my $para = "This is a super\eN{HYPHEN}long string. " x 20; |
| 739 |
\& my $fmt = Unicode::LineBreak\->new; |
| 740 |
\& print $fmt\->break($para), "\en"; |
| 741 |
.Ve |
| 742 |
.SS "℞ 42: Unicode text in DBM hashes, the tedious way" |
| 743 |
.IX Subsection "℞ 42: Unicode text in DBM hashes, the tedious way" |
| 744 |
Using a regular Perl string as a key or value for a DBM |
| 745 |
hash will trigger a wide character exception if any codepoints |
| 746 |
won’t fit into a byte. Here’s how to manually manage the translation: |
| 747 |
.PP |
| 748 |
.Vb 3 |
| 749 |
\& use DB_File; |
| 750 |
\& use Encode qw(encode decode); |
| 751 |
\& tie %dbhash, "DB_File", "pathname"; |
| 752 |
\& |
| 753 |
\& # STORE |
| 754 |
\& |
| 755 |
\& # assume $uni_key and $uni_value are abstract Unicode strings |
| 756 |
\& my $enc_key = encode("UTF\-8", $uni_key, 1); |
| 757 |
\& my $enc_value = encode("UTF\-8", $uni_value, 1); |
| 758 |
\& $dbhash{$enc_key} = $enc_value; |
| 759 |
\& |
| 760 |
\& # FETCH |
| 761 |
\& |
| 762 |
\& # assume $uni_key holds a normal Perl string (abstract Unicode) |
| 763 |
\& my $enc_key = encode("UTF\-8", $uni_key, 1); |
| 764 |
\& my $enc_value = $dbhash{$enc_key}; |
| 765 |
\& my $uni_value = decode("UTF\-8", $enc_value, 1); |
| 766 |
.Ve |
| 767 |
.SS "℞ 43: Unicode text in DBM hashes, the easy way" |
| 768 |
.IX Subsection "℞ 43: Unicode text in DBM hashes, the easy way" |
| 769 |
Here’s how to implicitly manage the translation; all encoding |
| 770 |
and decoding is done automatically, just as with streams that |
| 771 |
have a particular encoding attached to them: |
| 772 |
.PP |
| 773 |
.Vb 2 |
| 774 |
\& use DB_File; |
| 775 |
\& use DBM_Filter; |
| 776 |
\& |
| 777 |
\& my $dbobj = tie %dbhash, "DB_File", "pathname"; |
| 778 |
\& $dbobj\->Filter_Value("utf8"); # this is the magic bit |
| 779 |
\& |
| 780 |
\& # STORE |
| 781 |
\& |
| 782 |
\& # assume $uni_key and $uni_value are abstract Unicode strings |
| 783 |
\& $dbhash{$uni_key} = $uni_value; |
| 784 |
\& |
| 785 |
\& # FETCH |
| 786 |
\& |
| 787 |
\& # $uni_key holds a normal Perl string (abstract Unicode) |
| 788 |
\& my $uni_value = $dbhash{$uni_key}; |
| 789 |
.Ve |
| 790 |
.SS "℞ 44: PROGRAM: Demo of Unicode collation and printing" |
| 791 |
.IX Subsection "℞ 44: PROGRAM: Demo of Unicode collation and printing" |
| 792 |
Here’s a full program showing how to make use of locale-sensitive |
| 793 |
sorting, Unicode casing, and managing print widths when some of the |
| 794 |
characters take up zero or two columns, not just one column each time. |
| 795 |
When run, the following program produces this nicely aligned output: |
| 796 |
.PP |
| 797 |
.Vb 10 |
| 798 |
\& Crème Brûlée....... €2.00 |
| 799 |
\& Éclair............. €1.60 |
| 800 |
\& Fideuà............. €4.20 |
| 801 |
\& Hamburger.......... €6.00 |
| 802 |
\& Jamón Serrano...... €4.45 |
| 803 |
\& Linguiça........... €7.00 |
| 804 |
\& Pâté............... €4.15 |
| 805 |
\& Pears.............. €2.00 |
| 806 |
\& Pêches............. €2.25 |
| 807 |
\& Smørbrød........... €5.75 |
| 808 |
\& Spätzle............ €5.50 |
| 809 |
\& Xoriço............. €3.00 |
| 810 |
\& Γύρος.............. €6.50 |
| 811 |
\& 막걸리............. €4.00 |
| 812 |
\& おもち............. €2.65 |
| 813 |
\& お好み焼き......... €8.00 |
| 814 |
\& シュークリーム..... €1.85 |
| 815 |
\& 寿司............... €9.99 |
| 816 |
\& 包子............... €7.50 |
| 817 |
.Ve |
| 818 |
.PP |
| 819 |
Here's that program. |
| 820 |
.PP |
| 821 |
.Vb 10 |
| 822 |
\& #!/usr/bin/env perl |
| 823 |
\& # umenu \- demo sorting and printing of Unicode food |
| 824 |
\& # |
| 825 |
\& # (obligatory and increasingly long preamble) |
| 826 |
\& # |
| 827 |
\& use v5.36; |
| 828 |
\& use utf8; |
| 829 |
\& use warnings qw(FATAL utf8); # fatalize encoding faults |
| 830 |
\& use open qw(:std :encoding(UTF\-8)); # undeclared streams in UTF\-8 |
| 831 |
\& use charnames qw(:full :short); # unneeded in v5.16 |
| 832 |
\& |
| 833 |
\& # std modules |
| 834 |
\& use Unicode::Normalize; # std perl distro as of v5.8 |
| 835 |
\& use List::Util qw(max); # std perl distro as of v5.10 |
| 836 |
\& use Unicode::Collate::Locale; # std perl distro as of v5.14 |
| 837 |
\& |
| 838 |
\& # cpan modules |
| 839 |
\& use Unicode::GCString; # from CPAN |
| 840 |
\& |
| 841 |
\& my %price = ( |
| 842 |
\& "γύρος" => 6.50, # gyros |
| 843 |
\& "pears" => 2.00, # like um, pears |
| 844 |
\& "linguiça" => 7.00, # spicy sausage, Portuguese |
| 845 |
\& "xoriço" => 3.00, # chorizo sausage, Catalan |
| 846 |
\& "hamburger" => 6.00, # burgermeister meisterburger |
| 847 |
\& "éclair" => 1.60, # dessert, French |
| 848 |
\& "smørbrød" => 5.75, # sandwiches, Norwegian |
| 849 |
\& "spätzle" => 5.50, # Bayerisch noodles, little sparrows |
| 850 |
\& "包子" => 7.50, # bao1 zi5, steamed pork buns, Mandarin |
| 851 |
\& "jamón serrano" => 4.45, # country ham, Spanish |
| 852 |
\& "pêches" => 2.25, # peaches, French |
| 853 |
\& "シュークリーム" => 1.85, # cream\-filled pastry like eclair |
| 854 |
\& "막걸리" => 4.00, # makgeolli, Korean rice wine |
| 855 |
\& "寿司" => 9.99, # sushi, Japanese |
| 856 |
\& "おもち" => 2.65, # omochi, rice cakes, Japanese |
| 857 |
\& "crème brûlée" => 2.00, # crema catalana |
| 858 |
\& "fideuà" => 4.20, # more noodles, Valencian |
| 859 |
\& # (Catalan=fideuada) |
| 860 |
\& "pâté" => 4.15, # gooseliver paste, French |
| 861 |
\& "お好み焼き" => 8.00, # okonomiyaki, Japanese |
| 862 |
\& ); |
| 863 |
\& |
| 864 |
\& my $width = 5 + max map { colwidth($_) } keys %price; |
| 865 |
\& |
| 866 |
\& # So the Asian stuff comes out in an order that someone |
| 867 |
\& # who reads those scripts won\*(Aqt freak out over; the |
| 868 |
\& # CJK stuff will be in JIS X 0208 order that way. |
| 869 |
\& my $coll = Unicode::Collate::Locale\->new(locale => "ja"); |
| 870 |
\& |
| 871 |
\& for my $item ($coll\->sort(keys %price)) { |
| 872 |
\& print pad(entitle($item), $width, "."); |
| 873 |
\& printf " €%.2f\en", $price{$item}; |
| 874 |
\& } |
| 875 |
\& |
| 876 |
\& sub pad ($str, $width, $padchar) { |
| 877 |
\& return $str . ($padchar x ($width \- colwidth($str))); |
| 878 |
\& } |
| 879 |
\& |
| 880 |
\& sub colwidth ($str) { |
| 881 |
\& return Unicode::GCString\->new($str)\->columns; |
| 882 |
\& } |
| 883 |
\& |
| 884 |
\& sub entitle ($str) { |
| 885 |
\& $str =~ s{ (?=\epL)(\eS) (\eS*) } |
| 886 |
\& { ucfirst($1) . lc($2) }xge; |
| 887 |
\& return $str; |
| 888 |
\& } |
| 889 |
.Ve |
| 890 |
.SH "SEE ALSO" |
| 891 |
.IX Header "SEE ALSO" |
| 892 |
See these manpages, some of which are CPAN modules: |
| 893 |
perlunicode, perluniprops, |
| 894 |
perlre, perlrecharclass, |
| 895 |
perluniintro, perlunitut, perlunifaq, |
| 896 |
PerlIO, DB_File, DBM_Filter, DBM_Filter::utf8, |
| 897 |
Encode, Encode::Locale, |
| 898 |
Unicode::UCD, |
| 899 |
Unicode::Normalize, |
| 900 |
Unicode::GCString, Unicode::LineBreak, |
| 901 |
Unicode::Collate, Unicode::Collate::Locale, |
| 902 |
Unicode::Unihan, |
| 903 |
Unicode::CaseFold, |
| 904 |
Unicode::Tussle, |
| 905 |
Lingua::JA::Romanize::Japanese, |
| 906 |
Lingua::ZH::Romanize::Pinyin, |
| 907 |
Lingua::KO::Romanize::Hangul. |
| 908 |
.PP |
| 909 |
The Unicode::Tussle CPAN module includes many programs |
| 910 |
to help with working with Unicode, including |
| 911 |
these programs to fully or partly replace standard utilities: |
| 912 |
\&\fItcgrep\fR instead of \fIegrep\fR, |
| 913 |
\&\fIuniquote\fR instead of \fIcat \-v\fR or \fIhexdump\fR, |
| 914 |
\&\fIuniwc\fR instead of \fIwc\fR, |
| 915 |
\&\fIunilook\fR instead of \fIlook\fR, |
| 916 |
\&\fIunifmt\fR instead of \fIfmt\fR, |
| 917 |
and |
| 918 |
\&\fIucsort\fR instead of \fIsort\fR. |
| 919 |
For exploring Unicode character names and character properties, |
| 920 |
see its \fIuniprops\fR, \fIunichars\fR, and \fIuninames\fR programs. |
| 921 |
It also supplies these programs, all of which are general filters that do Unicode-y things: |
| 922 |
\&\fIunititle\fR and \fIunicaps\fR; |
| 923 |
\&\fIuniwide\fR and \fIuninarrow\fR; |
| 924 |
\&\fIunisupers\fR and \fIunisubs\fR; |
| 925 |
\&\fInfd\fR, \fInfc\fR, \fInfkd\fR, and \fInfkc\fR; |
| 926 |
and \fIuc\fR, \fIlc\fR, and \fItc\fR. |
| 927 |
.PP |
| 928 |
Finally, see the published Unicode Standard (page numbers are from version |
| 929 |
6.0.0), including these specific annexes and technical reports: |
| 930 |
.IP "§3.13 Default Case Algorithms, page 113; §4.2 Case, pages 120–122; Case Mappings, page 166–172, especially Caseless Matching starting on page 170." 4 |
| 931 |
.IX Item "§3.13 Default Case Algorithms, page 113; §4.2 Case, pages 120–122; Case Mappings, page 166–172, especially Caseless Matching starting on page 170." |
| 932 |
.PD 0 |
| 933 |
.IP "UAX #44: Unicode Character Database" 4 |
| 934 |
.IX Item "UAX #44: Unicode Character Database" |
| 935 |
.IP "UTS #18: Unicode Regular Expressions" 4 |
| 936 |
.IX Item "UTS #18: Unicode Regular Expressions" |
| 937 |
.IP "UAX #15: Unicode Normalization Forms" 4 |
| 938 |
.IX Item "UAX #15: Unicode Normalization Forms" |
| 939 |
.IP "UTS #10: Unicode Collation Algorithm" 4 |
| 940 |
.IX Item "UTS #10: Unicode Collation Algorithm" |
| 941 |
.IP "UAX #29: Unicode Text Segmentation" 4 |
| 942 |
.IX Item "UAX #29: Unicode Text Segmentation" |
| 943 |
.IP "UAX #14: Unicode Line Breaking Algorithm" 4 |
| 944 |
.IX Item "UAX #14: Unicode Line Breaking Algorithm" |
| 945 |
.IP "UAX #11: East Asian Width" 4 |
| 946 |
.IX Item "UAX #11: East Asian Width" |
| 947 |
.PD |
| 948 |
.SH AUTHOR |
| 949 |
.IX Header "AUTHOR" |
| 950 |
Tom Christiansen <[email protected]> wrote this, with occasional |
| 951 |
kibbitzing from Larry Wall and Jeffrey Friedl in the background. |
| 952 |
.SH "COPYRIGHT AND LICENCE" |
| 953 |
.IX Header "COPYRIGHT AND LICENCE" |
| 954 |
Copyright © 2012 Tom Christiansen. |
| 955 |
.PP |
| 956 |
This program is free software; you may redistribute it and/or modify it |
| 957 |
under the same terms as Perl itself. |
| 958 |
.PP |
| 959 |
Most of these examples taken from the current edition of the “Camel Book”; |
| 960 |
that is, from the 4ᵗʰ Edition of \fIProgramming Perl\fR, Copyright © 2012 Tom |
| 961 |
Christiansen <et al.>, 2012\-02\-13 by O’Reilly Media. The code itself is |
| 962 |
freely redistributable, and you are encouraged to transplant, fold, |
| 963 |
spindle, and mutilate any of the examples in this manpage however you please |
| 964 |
for inclusion into your own programs without any encumbrance whatsoever. |
| 965 |
Acknowledgement via code comment is polite but not required. |
| 966 |
.SH "REVISION HISTORY" |
| 967 |
.IX Header "REVISION HISTORY" |
| 968 |
v1.0.0 – first public release, 2012\-02\-27 |