| 1 |
.\" -*- mode: troff; coding: utf-8 -*- |
| 2 |
.\" Automatically generated by Pod::Man 5.0102 (Pod::Simple 3.45) |
| 3 |
.\" |
| 4 |
.\" Standard preamble: |
| 5 |
.\" ======================================================================== |
| 6 |
.de Sp \" Vertical space (when we can't use .PP) |
| 7 |
.if t .sp .5v |
| 8 |
.if n .sp |
| 9 |
.. |
| 10 |
.de Vb \" Begin verbatim text |
| 11 |
.ft CW |
| 12 |
.nf |
| 13 |
.ne \\$1 |
| 14 |
.. |
| 15 |
.de Ve \" End verbatim text |
| 16 |
.ft R |
| 17 |
.fi |
| 18 |
.. |
| 19 |
.\" \*(C` and \*(C' are quotes in nroff, nothing in troff, for use with C<>. |
| 20 |
.ie n \{\ |
| 21 |
. ds C` "" |
| 22 |
. ds C' "" |
| 23 |
'br\} |
| 24 |
.el\{\ |
| 25 |
. ds C` |
| 26 |
. ds C' |
| 27 |
'br\} |
| 28 |
.\" |
| 29 |
.\" Escape single quotes in literal strings from groff's Unicode transform. |
| 30 |
.ie \n(.g .ds Aq \(aq |
| 31 |
.el .ds Aq ' |
| 32 |
.\" |
| 33 |
.\" If the F register is >0, we'll generate index entries on stderr for |
| 34 |
.\" titles (.TH), headers (.SH), subsections (.SS), items (.Ip), and index |
| 35 |
.\" entries marked with X<> in POD. Of course, you'll have to process the |
| 36 |
.\" output yourself in some meaningful fashion. |
| 37 |
.\" |
| 38 |
.\" Avoid warning from groff about undefined register 'F'. |
| 39 |
.de IX |
| 40 |
.. |
| 41 |
.nr rF 0 |
| 42 |
.if \n(.g .if rF .nr rF 1 |
| 43 |
.if (\n(rF:(\n(.g==0)) \{\ |
| 44 |
. if \nF \{\ |
| 45 |
. de IX |
| 46 |
. tm Index:\\$1\t\\n%\t"\\$2" |
| 47 |
.. |
| 48 |
. if !\nF==2 \{\ |
| 49 |
. nr % 0 |
| 50 |
. nr F 2 |
| 51 |
. \} |
| 52 |
. \} |
| 53 |
.\} |
| 54 |
.rr rF |
| 55 |
.\" ======================================================================== |
| 56 |
.\" |
| 57 |
.IX Title "Encode 3perl" |
| 58 |
.TH Encode 3perl 2025-11-16 "perl v5.40.1" "Perl Programmers Reference Guide" |
| 59 |
.\" For nroff, turn off justification. Always turn off hyphenation; it makes |
| 60 |
.\" way too many mistakes in technical documents. |
| 61 |
.if n .ad l |
| 62 |
.nh |
| 63 |
.SH NAME |
| 64 |
Encode \- character encodings in Perl |
| 65 |
.SH SYNOPSIS |
| 66 |
.IX Header "SYNOPSIS" |
| 67 |
.Vb 3 |
| 68 |
\& use Encode qw(decode encode); |
| 69 |
\& $characters = decode(\*(AqUTF\-8\*(Aq, $octets, Encode::FB_CROAK); |
| 70 |
\& $octets = encode(\*(AqUTF\-8\*(Aq, $characters, Encode::FB_CROAK); |
| 71 |
.Ve |
| 72 |
.SS "Table of Contents" |
| 73 |
.IX Subsection "Table of Contents" |
| 74 |
Encode consists of a collection of modules whose details are too extensive |
| 75 |
to fit in one document. This one itself explains the top-level APIs |
| 76 |
and general topics at a glance. For other topics and more details, |
| 77 |
see the documentation for these modules: |
| 78 |
.IP "Encode::Alias \- Alias definitions to encodings" 2 |
| 79 |
.IX Item "Encode::Alias - Alias definitions to encodings" |
| 80 |
.PD 0 |
| 81 |
.IP "Encode::Encoding \- Encode Implementation Base Class" 2 |
| 82 |
.IX Item "Encode::Encoding - Encode Implementation Base Class" |
| 83 |
.IP "Encode::Supported \- List of Supported Encodings" 2 |
| 84 |
.IX Item "Encode::Supported - List of Supported Encodings" |
| 85 |
.IP "Encode::CN \- Simplified Chinese Encodings" 2 |
| 86 |
.IX Item "Encode::CN - Simplified Chinese Encodings" |
| 87 |
.IP "Encode::JP \- Japanese Encodings" 2 |
| 88 |
.IX Item "Encode::JP - Japanese Encodings" |
| 89 |
.IP "Encode::KR \- Korean Encodings" 2 |
| 90 |
.IX Item "Encode::KR - Korean Encodings" |
| 91 |
.IP "Encode::TW \- Traditional Chinese Encodings" 2 |
| 92 |
.IX Item "Encode::TW - Traditional Chinese Encodings" |
| 93 |
.PD |
| 94 |
.SH DESCRIPTION |
| 95 |
.IX Header "DESCRIPTION" |
| 96 |
The \f(CW\*(C`Encode\*(C'\fR module provides the interface between Perl strings |
| 97 |
and the rest of the system. Perl strings are sequences of |
| 98 |
\&\fIcharacters\fR. |
| 99 |
.PP |
| 100 |
The repertoire of characters that Perl can represent is a superset of those |
| 101 |
defined by the Unicode Consortium. On most platforms the ordinal |
| 102 |
values of a character as returned by \f(CWord(\fR\f(CIS\fR\f(CW)\fR is the \fIUnicode |
| 103 |
codepoint\fR for that character. The exceptions are platforms where |
| 104 |
the legacy encoding is some variant of EBCDIC rather than a superset |
| 105 |
of ASCII; see perlebcdic. |
| 106 |
.PP |
| 107 |
During recent history, data is moved around a computer in 8\-bit chunks, |
| 108 |
often called "bytes" but also known as "octets" in standards documents. |
| 109 |
Perl is widely used to manipulate data of many types: not only strings of |
| 110 |
characters representing human or computer languages, but also "binary" |
| 111 |
data, being the machine's representation of numbers, pixels in an image, or |
| 112 |
just about anything. |
| 113 |
.PP |
| 114 |
When Perl is processing "binary data", the programmer wants Perl to |
| 115 |
process "sequences of bytes". This is not a problem for Perl: because a |
| 116 |
byte has 256 possible values, it easily fits in Perl's much larger |
| 117 |
"logical character". |
| 118 |
.PP |
| 119 |
This document mostly explains the \fIhow\fR. perlunitut and perlunifaq |
| 120 |
explain the \fIwhy\fR. |
| 121 |
.SS TERMINOLOGY |
| 122 |
.IX Subsection "TERMINOLOGY" |
| 123 |
\fIcharacter\fR |
| 124 |
.IX Subsection "character" |
| 125 |
.PP |
| 126 |
A character in the range 0 .. 2**32\-1 (or more); |
| 127 |
what Perl's strings are made of. |
| 128 |
.PP |
| 129 |
\fIbyte\fR |
| 130 |
.IX Subsection "byte" |
| 131 |
.PP |
| 132 |
A character in the range 0..255; |
| 133 |
a special case of a Perl character. |
| 134 |
.PP |
| 135 |
\fIoctet\fR |
| 136 |
.IX Subsection "octet" |
| 137 |
.PP |
| 138 |
8 bits of data, with ordinal values 0..255; |
| 139 |
term for bytes passed to or from a non-Perl context, such as a disk file, |
| 140 |
standard I/O stream, database, command-line argument, environment variable, |
| 141 |
socket etc. |
| 142 |
.SH "THE PERL ENCODING API" |
| 143 |
.IX Header "THE PERL ENCODING API" |
| 144 |
.SS "Basic methods" |
| 145 |
.IX Subsection "Basic methods" |
| 146 |
\fIencode\fR |
| 147 |
.IX Subsection "encode" |
| 148 |
.PP |
| 149 |
.Vb 1 |
| 150 |
\& $octets = encode(ENCODING, STRING[, CHECK]) |
| 151 |
.Ve |
| 152 |
.PP |
| 153 |
Encodes the scalar value \fISTRING\fR from Perl's internal form into |
| 154 |
\&\fIENCODING\fR and returns a sequence of octets. \fIENCODING\fR can be either a |
| 155 |
canonical name or an alias. For encoding names and aliases, see |
| 156 |
"Defining Aliases". For CHECK, see "Handling Malformed Data". |
| 157 |
.PP |
| 158 |
\&\fBCAVEAT\fR: the input scalar \fISTRING\fR might be modified in-place depending |
| 159 |
on what is set in CHECK. See "LEAVE_SRC" if you want your inputs to be |
| 160 |
left unchanged. |
| 161 |
.PP |
| 162 |
For example, to convert a string from Perl's internal format into |
| 163 |
ISO\-8859\-1, also known as Latin1: |
| 164 |
.PP |
| 165 |
.Vb 1 |
| 166 |
\& $octets = encode("iso\-8859\-1", $string); |
| 167 |
.Ve |
| 168 |
.PP |
| 169 |
\&\fBCAVEAT\fR: When you run \f(CW\*(C`$octets = encode("UTF\-8", $string)\*(C'\fR, then |
| 170 |
\&\f(CW$octets\fR \fImight not be equal to\fR \f(CW$string\fR. Though both contain the |
| 171 |
same data, the UTF8 flag for \f(CW$octets\fR is \fIalways\fR off. When you |
| 172 |
encode anything, the UTF8 flag on the result is always off, even when it |
| 173 |
contains a completely valid UTF\-8 string. See "The UTF8 flag" below. |
| 174 |
.PP |
| 175 |
If the \f(CW$string\fR is \f(CW\*(C`undef\*(C'\fR, then \f(CW\*(C`undef\*(C'\fR is returned. |
| 176 |
.PP |
| 177 |
\&\f(CW\*(C`str2bytes\*(C'\fR may be used as an alias for \f(CW\*(C`encode\*(C'\fR. |
| 178 |
.PP |
| 179 |
\fIdecode\fR |
| 180 |
.IX Subsection "decode" |
| 181 |
.PP |
| 182 |
.Vb 1 |
| 183 |
\& $string = decode(ENCODING, OCTETS[, CHECK]) |
| 184 |
.Ve |
| 185 |
.PP |
| 186 |
This function returns the string that results from decoding the scalar |
| 187 |
value \fIOCTETS\fR, assumed to be a sequence of octets in \fIENCODING\fR, into |
| 188 |
Perl's internal form. As with \fBencode()\fR, |
| 189 |
\&\fIENCODING\fR can be either a canonical name or an alias. For encoding names |
| 190 |
and aliases, see "Defining Aliases"; for \fICHECK\fR, see "Handling |
| 191 |
Malformed Data". |
| 192 |
.PP |
| 193 |
\&\fBCAVEAT\fR: the input scalar \fIOCTETS\fR might be modified in-place depending |
| 194 |
on what is set in CHECK. See "LEAVE_SRC" if you want your inputs to be |
| 195 |
left unchanged. |
| 196 |
.PP |
| 197 |
For example, to convert ISO\-8859\-1 data into a string in Perl's |
| 198 |
internal format: |
| 199 |
.PP |
| 200 |
.Vb 1 |
| 201 |
\& $string = decode("iso\-8859\-1", $octets); |
| 202 |
.Ve |
| 203 |
.PP |
| 204 |
\&\fBCAVEAT\fR: When you run \f(CW\*(C`$string = decode("UTF\-8", $octets)\*(C'\fR, then \f(CW$string\fR |
| 205 |
\&\fImight not be equal to\fR \f(CW$octets\fR. Though both contain the same data, the |
| 206 |
UTF8 flag for \f(CW$string\fR is on. See "The UTF8 flag" |
| 207 |
below. |
| 208 |
.PP |
| 209 |
If the \f(CW$string\fR is \f(CW\*(C`undef\*(C'\fR, then \f(CW\*(C`undef\*(C'\fR is returned. |
| 210 |
.PP |
| 211 |
\&\f(CW\*(C`bytes2str\*(C'\fR may be used as an alias for \f(CW\*(C`decode\*(C'\fR. |
| 212 |
.PP |
| 213 |
\fIfind_encoding\fR |
| 214 |
.IX Subsection "find_encoding" |
| 215 |
.PP |
| 216 |
.Vb 1 |
| 217 |
\& [$obj =] find_encoding(ENCODING) |
| 218 |
.Ve |
| 219 |
.PP |
| 220 |
Returns the \fIencoding object\fR corresponding to \fIENCODING\fR. Returns |
| 221 |
\&\f(CW\*(C`undef\*(C'\fR if no matching \fIENCODING\fR is find. The returned object is |
| 222 |
what does the actual encoding or decoding. |
| 223 |
.PP |
| 224 |
.Vb 1 |
| 225 |
\& $string = decode($name, $bytes); |
| 226 |
.Ve |
| 227 |
.PP |
| 228 |
is in fact |
| 229 |
.PP |
| 230 |
.Vb 5 |
| 231 |
\& $string = do { |
| 232 |
\& $obj = find_encoding($name); |
| 233 |
\& croak qq(encoding "$name" not found) unless ref $obj; |
| 234 |
\& $obj\->decode($bytes); |
| 235 |
\& }; |
| 236 |
.Ve |
| 237 |
.PP |
| 238 |
with more error checking. |
| 239 |
.PP |
| 240 |
You can therefore save time by reusing this object as follows; |
| 241 |
.PP |
| 242 |
.Vb 5 |
| 243 |
\& my $enc = find_encoding("iso\-8859\-1"); |
| 244 |
\& while(<>) { |
| 245 |
\& my $string = $enc\->decode($_); |
| 246 |
\& ... # now do something with $string; |
| 247 |
\& } |
| 248 |
.Ve |
| 249 |
.PP |
| 250 |
Besides "decode" and "encode", other methods are |
| 251 |
available as well. For instance, \f(CWname()\fR returns the canonical |
| 252 |
name of the encoding object. |
| 253 |
.PP |
| 254 |
.Vb 1 |
| 255 |
\& find_encoding("latin1")\->name; # iso\-8859\-1 |
| 256 |
.Ve |
| 257 |
.PP |
| 258 |
See Encode::Encoding for details. |
| 259 |
.PP |
| 260 |
\fIfind_mime_encoding\fR |
| 261 |
.IX Subsection "find_mime_encoding" |
| 262 |
.PP |
| 263 |
.Vb 1 |
| 264 |
\& [$obj =] find_mime_encoding(MIME_ENCODING) |
| 265 |
.Ve |
| 266 |
.PP |
| 267 |
Returns the \fIencoding object\fR corresponding to \fIMIME_ENCODING\fR. Acts |
| 268 |
same as \f(CWfind_encoding()\fR but \f(CWmime_name()\fR of returned object must |
| 269 |
match to \fIMIME_ENCODING\fR. So as opposite of \f(CWfind_encoding()\fR |
| 270 |
canonical names and aliases are not used when searching for object. |
| 271 |
.PP |
| 272 |
.Vb 4 |
| 273 |
\& find_mime_encoding("utf8"); # returns undef because "utf8" is not a valid MIME_ENCODING |
| 274 |
\& find_mime_encoding("utf\-8"); # returns encode object "utf\-8\-strict" |
| 275 |
\& find_mime_encoding("UTF\-8"); # same as "utf\-8" because MIME_ENCODING is case insensitive |
| 276 |
\& find_mime_encoding("utf\-8\-strict"); returns undef because "utf\-8\-strict" is not a valid MIME_ENCODING |
| 277 |
.Ve |
| 278 |
.PP |
| 279 |
\fIfrom_to\fR |
| 280 |
.IX Subsection "from_to" |
| 281 |
.PP |
| 282 |
.Vb 1 |
| 283 |
\& [$length =] from_to($octets, FROM_ENC, TO_ENC [, CHECK]) |
| 284 |
.Ve |
| 285 |
.PP |
| 286 |
Converts \fIin-place\fR data between two encodings. The data in \f(CW$octets\fR |
| 287 |
must be encoded as octets and \fInot\fR as characters in Perl's internal |
| 288 |
format. For example, to convert ISO\-8859\-1 data into Microsoft's CP1250 |
| 289 |
encoding: |
| 290 |
.PP |
| 291 |
.Vb 1 |
| 292 |
\& from_to($octets, "iso\-8859\-1", "cp1250"); |
| 293 |
.Ve |
| 294 |
.PP |
| 295 |
and to convert it back: |
| 296 |
.PP |
| 297 |
.Vb 1 |
| 298 |
\& from_to($octets, "cp1250", "iso\-8859\-1"); |
| 299 |
.Ve |
| 300 |
.PP |
| 301 |
Because the conversion happens in place, the data to be |
| 302 |
converted cannot be a string constant: it must be a scalar variable. |
| 303 |
.PP |
| 304 |
\&\f(CWfrom_to()\fR returns the length of the converted string in octets on success, |
| 305 |
and \f(CW\*(C`undef\*(C'\fR on error. |
| 306 |
.PP |
| 307 |
\&\fBCAVEAT\fR: The following operations may look the same, but are not: |
| 308 |
.PP |
| 309 |
.Vb 2 |
| 310 |
\& from_to($data, "iso\-8859\-1", "UTF\-8"); #1 |
| 311 |
\& $data = decode("iso\-8859\-1", $data); #2 |
| 312 |
.Ve |
| 313 |
.PP |
| 314 |
Both #1 and #2 make \f(CW$data\fR consist of a completely valid UTF\-8 string, |
| 315 |
but only #2 turns the UTF8 flag on. #1 is equivalent to: |
| 316 |
.PP |
| 317 |
.Vb 1 |
| 318 |
\& $data = encode("UTF\-8", decode("iso\-8859\-1", $data)); |
| 319 |
.Ve |
| 320 |
.PP |
| 321 |
See "The UTF8 flag" below. |
| 322 |
.PP |
| 323 |
Also note that: |
| 324 |
.PP |
| 325 |
.Vb 1 |
| 326 |
\& from_to($octets, $from, $to, $check); |
| 327 |
.Ve |
| 328 |
.PP |
| 329 |
is equivalent to: |
| 330 |
.PP |
| 331 |
.Vb 1 |
| 332 |
\& $octets = encode($to, decode($from, $octets), $check); |
| 333 |
.Ve |
| 334 |
.PP |
| 335 |
Yes, it does \fInot\fR respect the \f(CW$check\fR during decoding. It is |
| 336 |
deliberately done that way. If you need minute control, use \f(CW\*(C`decode\*(C'\fR |
| 337 |
followed by \f(CW\*(C`encode\*(C'\fR as follows: |
| 338 |
.PP |
| 339 |
.Vb 1 |
| 340 |
\& $octets = encode($to, decode($from, $octets, $check_from), $check_to); |
| 341 |
.Ve |
| 342 |
.PP |
| 343 |
\fIencode_utf8\fR |
| 344 |
.IX Subsection "encode_utf8" |
| 345 |
.PP |
| 346 |
.Vb 1 |
| 347 |
\& $octets = encode_utf8($string); |
| 348 |
.Ve |
| 349 |
.PP |
| 350 |
\&\fBWARNING\fR: This function can produce invalid UTF\-8! |
| 351 |
Do not use it for data exchange. |
| 352 |
Unless you want Perl's older "lax" mode, prefer |
| 353 |
\&\f(CW\*(C`$octets = encode("UTF\-8", $string)\*(C'\fR. |
| 354 |
.PP |
| 355 |
Equivalent to \f(CW\*(C`$octets = encode("utf8", $string)\*(C'\fR. The characters in |
| 356 |
\&\f(CW$string\fR are encoded in Perl's internal format, and the result is returned |
| 357 |
as a sequence of octets. Because all possible characters in Perl have a |
| 358 |
(loose, not strict) utf8 representation, this function cannot fail. |
| 359 |
.PP |
| 360 |
\fIdecode_utf8\fR |
| 361 |
.IX Subsection "decode_utf8" |
| 362 |
.PP |
| 363 |
.Vb 1 |
| 364 |
\& $string = decode_utf8($octets [, CHECK]); |
| 365 |
.Ve |
| 366 |
.PP |
| 367 |
\&\fBWARNING\fR: This function accepts invalid UTF\-8! |
| 368 |
Do not use it for data exchange. |
| 369 |
Unless you want Perl's older "lax" mode, prefer |
| 370 |
\&\f(CW\*(C`$string = decode("UTF\-8", $octets [, CHECK])\*(C'\fR. |
| 371 |
.PP |
| 372 |
Equivalent to \f(CW\*(C`$string = decode("utf8", $octets [, CHECK])\*(C'\fR. |
| 373 |
The sequence of octets represented by \f(CW$octets\fR is decoded |
| 374 |
from (loose, not strict) utf8 into a sequence of logical characters. |
| 375 |
Because not all sequences of octets are valid not strict utf8, |
| 376 |
it is quite possible for this function to fail. |
| 377 |
For CHECK, see "Handling Malformed Data". |
| 378 |
.PP |
| 379 |
\&\fBCAVEAT\fR: the input \fR\f(CI$octets\fR\fI\fR might be modified in-place depending on |
| 380 |
what is set in CHECK. See "LEAVE_SRC" if you want your inputs to be |
| 381 |
left unchanged. |
| 382 |
.SS "Listing available encodings" |
| 383 |
.IX Subsection "Listing available encodings" |
| 384 |
.Vb 2 |
| 385 |
\& use Encode; |
| 386 |
\& @list = Encode\->encodings(); |
| 387 |
.Ve |
| 388 |
.PP |
| 389 |
Returns a list of canonical names of available encodings that have already |
| 390 |
been loaded. To get a list of all available encodings including those that |
| 391 |
have not yet been loaded, say: |
| 392 |
.PP |
| 393 |
.Vb 1 |
| 394 |
\& @all_encodings = Encode\->encodings(":all"); |
| 395 |
.Ve |
| 396 |
.PP |
| 397 |
Or you can give the name of a specific module: |
| 398 |
.PP |
| 399 |
.Vb 1 |
| 400 |
\& @with_jp = Encode\->encodings("Encode::JP"); |
| 401 |
.Ve |
| 402 |
.PP |
| 403 |
When "\f(CW\*(C`::\*(C'\fR" is not in the name, "\f(CW\*(C`Encode::\*(C'\fR" is assumed. |
| 404 |
.PP |
| 405 |
.Vb 1 |
| 406 |
\& @ebcdic = Encode\->encodings("EBCDIC"); |
| 407 |
.Ve |
| 408 |
.PP |
| 409 |
To find out in detail which encodings are supported by this package, |
| 410 |
see Encode::Supported. |
| 411 |
.SS "Defining Aliases" |
| 412 |
.IX Subsection "Defining Aliases" |
| 413 |
To add a new alias to a given encoding, use: |
| 414 |
.PP |
| 415 |
.Vb 3 |
| 416 |
\& use Encode; |
| 417 |
\& use Encode::Alias; |
| 418 |
\& define_alias(NEWNAME => ENCODING); |
| 419 |
.Ve |
| 420 |
.PP |
| 421 |
After that, \fINEWNAME\fR can be used as an alias for \fIENCODING\fR. |
| 422 |
\&\fIENCODING\fR may be either the name of an encoding or an |
| 423 |
\&\fIencoding object\fR. |
| 424 |
.PP |
| 425 |
Before you do that, first make sure the alias is nonexistent using |
| 426 |
\&\f(CWresolve_alias()\fR, which returns the canonical name thereof. |
| 427 |
For example: |
| 428 |
.PP |
| 429 |
.Vb 3 |
| 430 |
\& Encode::resolve_alias("latin1") eq "iso\-8859\-1" # true |
| 431 |
\& Encode::resolve_alias("iso\-8859\-12") # false; nonexistent |
| 432 |
\& Encode::resolve_alias($name) eq $name # true if $name is canonical |
| 433 |
.Ve |
| 434 |
.PP |
| 435 |
\&\f(CWresolve_alias()\fR does not need \f(CW\*(C`use Encode::Alias\*(C'\fR; it can be |
| 436 |
imported via \f(CW\*(C`use Encode qw(resolve_alias)\*(C'\fR. |
| 437 |
.PP |
| 438 |
See Encode::Alias for details. |
| 439 |
.SS "Finding IANA Character Set Registry names" |
| 440 |
.IX Subsection "Finding IANA Character Set Registry names" |
| 441 |
The canonical name of a given encoding does not necessarily agree with |
| 442 |
IANA Character Set Registry, commonly seen as \f(CW\*(C`Content\-Type: |
| 443 |
text/plain; charset=\fR\f(CIWHATEVER\fR\f(CW\*(C'\fR. For most cases, the canonical name |
| 444 |
works, but sometimes it does not, most notably with "utf\-8\-strict". |
| 445 |
.PP |
| 446 |
As of \f(CW\*(C`Encode\*(C'\fR version 2.21, a new method \f(CWmime_name()\fR is therefore added. |
| 447 |
.PP |
| 448 |
.Vb 4 |
| 449 |
\& use Encode; |
| 450 |
\& my $enc = find_encoding("UTF\-8"); |
| 451 |
\& warn $enc\->name; # utf\-8\-strict |
| 452 |
\& warn $enc\->mime_name; # UTF\-8 |
| 453 |
.Ve |
| 454 |
.PP |
| 455 |
See also: Encode::Encoding |
| 456 |
.SH "Encoding via PerlIO" |
| 457 |
.IX Header "Encoding via PerlIO" |
| 458 |
If your perl supports \f(CW\*(C`PerlIO\*(C'\fR (which is the default), you can use a |
| 459 |
\&\f(CW\*(C`PerlIO\*(C'\fR layer to decode and encode directly via a filehandle. The |
| 460 |
following two examples are fully identical in functionality: |
| 461 |
.PP |
| 462 |
.Vb 10 |
| 463 |
\& ### Version 1 via PerlIO |
| 464 |
\& open(INPUT, "< :encoding(shiftjis)", $infile) |
| 465 |
\& || die "Can\*(Aqt open < $infile for reading: $!"; |
| 466 |
\& open(OUTPUT, "> :encoding(euc\-jp)", $outfile) |
| 467 |
\& || die "Can\*(Aqt open > $output for writing: $!"; |
| 468 |
\& while (<INPUT>) { # auto decodes $_ |
| 469 |
\& print OUTPUT; # auto encodes $_ |
| 470 |
\& } |
| 471 |
\& close(INPUT) || die "can\*(Aqt close $infile: $!"; |
| 472 |
\& close(OUTPUT) || die "can\*(Aqt close $outfile: $!"; |
| 473 |
\& |
| 474 |
\& ### Version 2 via from_to() |
| 475 |
\& open(INPUT, "< :raw", $infile) |
| 476 |
\& || die "Can\*(Aqt open < $infile for reading: $!"; |
| 477 |
\& open(OUTPUT, "> :raw", $outfile) |
| 478 |
\& || die "Can\*(Aqt open > $output for writing: $!"; |
| 479 |
\& |
| 480 |
\& while (<INPUT>) { |
| 481 |
\& from_to($_, "shiftjis", "euc\-jp", 1); # switch encoding |
| 482 |
\& print OUTPUT; # emit raw (but properly encoded) data |
| 483 |
\& } |
| 484 |
\& close(INPUT) || die "can\*(Aqt close $infile: $!"; |
| 485 |
\& close(OUTPUT) || die "can\*(Aqt close $outfile: $!"; |
| 486 |
.Ve |
| 487 |
.PP |
| 488 |
In the first version above, you let the appropriate encoding layer |
| 489 |
handle the conversion. In the second, you explicitly translate |
| 490 |
from one encoding to the other. |
| 491 |
.PP |
| 492 |
Unfortunately, it may be that encodings are not \f(CW\*(C`PerlIO\*(C'\fR\-savvy. You can check |
| 493 |
to see whether your encoding is supported by \f(CW\*(C`PerlIO\*(C'\fR by invoking the |
| 494 |
\&\f(CW\*(C`perlio_ok\*(C'\fR method on it: |
| 495 |
.PP |
| 496 |
.Vb 2 |
| 497 |
\& Encode::perlio_ok("hz"); # false |
| 498 |
\& find_encoding("euc\-cn")\->perlio_ok; # true wherever PerlIO is available |
| 499 |
\& |
| 500 |
\& use Encode qw(perlio_ok); # imported upon request |
| 501 |
\& perlio_ok("euc\-jp") |
| 502 |
.Ve |
| 503 |
.PP |
| 504 |
Fortunately, all encodings that come with \f(CW\*(C`Encode\*(C'\fR core are \f(CW\*(C`PerlIO\*(C'\fR\-savvy |
| 505 |
except for \f(CW\*(C`hz\*(C'\fR and \f(CW\*(C`ISO\-2022\-kr\*(C'\fR. For the gory details, see |
| 506 |
Encode::Encoding and Encode::PerlIO. |
| 507 |
.SH "Handling Malformed Data" |
| 508 |
.IX Header "Handling Malformed Data" |
| 509 |
The optional \fICHECK\fR argument tells \f(CW\*(C`Encode\*(C'\fR what to do when |
| 510 |
encountering malformed data. Without \fICHECK\fR, \f(CW\*(C`Encode::FB_DEFAULT\*(C'\fR |
| 511 |
(== 0) is assumed. |
| 512 |
.PP |
| 513 |
As of version 2.12, \f(CW\*(C`Encode\*(C'\fR supports coderef values for \f(CW\*(C`CHECK\*(C'\fR; |
| 514 |
see below. |
| 515 |
.PP |
| 516 |
\&\fBNOTE:\fR Not all encodings support this feature. |
| 517 |
Some encodings ignore the \fICHECK\fR argument. For example, |
| 518 |
Encode::Unicode ignores \fICHECK\fR and it always croaks on error. |
| 519 |
.SS "List of \fICHECK\fP values" |
| 520 |
.IX Subsection "List of CHECK values" |
| 521 |
\fIFB_DEFAULT\fR |
| 522 |
.IX Subsection "FB_DEFAULT" |
| 523 |
.PP |
| 524 |
.Vb 1 |
| 525 |
\& CHECK = Encode::FB_DEFAULT ( == 0) |
| 526 |
.Ve |
| 527 |
.PP |
| 528 |
If \fICHECK\fR is 0, encoding and decoding replace any malformed character |
| 529 |
with a \fIsubstitution character\fR. When you encode, \fISUBCHAR\fR is used. |
| 530 |
When you decode, the Unicode REPLACEMENT CHARACTER, code point U+FFFD, is |
| 531 |
used. If the data is supposed to be UTF\-8, an optional lexical warning of |
| 532 |
warning category \f(CW"utf8"\fR is given. |
| 533 |
.PP |
| 534 |
\fIFB_CROAK\fR |
| 535 |
.IX Subsection "FB_CROAK" |
| 536 |
.PP |
| 537 |
.Vb 1 |
| 538 |
\& CHECK = Encode::FB_CROAK ( == 1) |
| 539 |
.Ve |
| 540 |
.PP |
| 541 |
If \fICHECK\fR is 1, methods immediately die with an error |
| 542 |
message. Therefore, when \fICHECK\fR is 1, you should trap |
| 543 |
exceptions with \f(CW\*(C`eval{}\*(C'\fR, unless you really want to let it \f(CW\*(C`die\*(C'\fR. |
| 544 |
.PP |
| 545 |
\fIFB_QUIET\fR |
| 546 |
.IX Subsection "FB_QUIET" |
| 547 |
.PP |
| 548 |
.Vb 1 |
| 549 |
\& CHECK = Encode::FB_QUIET |
| 550 |
.Ve |
| 551 |
.PP |
| 552 |
If \fICHECK\fR is set to \f(CW\*(C`Encode::FB_QUIET\*(C'\fR, encoding and decoding immediately |
| 553 |
return the portion of the data that has been processed so far when an |
| 554 |
error occurs. The data argument is overwritten with everything |
| 555 |
after that point; that is, the unprocessed portion of the data. This is |
| 556 |
handy when you have to call \f(CW\*(C`decode\*(C'\fR repeatedly in the case where your |
| 557 |
source data may contain partial multi-byte character sequences, |
| 558 |
(that is, you are reading with a fixed-width buffer). Here's some sample |
| 559 |
code to do exactly that: |
| 560 |
.PP |
| 561 |
.Vb 5 |
| 562 |
\& my($buffer, $string) = ("", ""); |
| 563 |
\& while (read($fh, $buffer, 256, length($buffer))) { |
| 564 |
\& $string .= decode($encoding, $buffer, Encode::FB_QUIET); |
| 565 |
\& # $buffer now contains the unprocessed partial character |
| 566 |
\& } |
| 567 |
.Ve |
| 568 |
.PP |
| 569 |
\fIFB_WARN\fR |
| 570 |
.IX Subsection "FB_WARN" |
| 571 |
.PP |
| 572 |
.Vb 1 |
| 573 |
\& CHECK = Encode::FB_WARN |
| 574 |
.Ve |
| 575 |
.PP |
| 576 |
This is the same as \f(CW\*(C`FB_QUIET\*(C'\fR above, except that instead of being silent |
| 577 |
on errors, it issues a warning. This is handy for when you are debugging. |
| 578 |
.PP |
| 579 |
\&\fBCAVEAT\fR: All warnings from Encode module are reported, independently of |
| 580 |
pragma warnings settings. If you want to follow settings of |
| 581 |
lexical warnings configured by pragma warnings then append |
| 582 |
also check value \f(CW\*(C`ENCODE::ONLY_PRAGMA_WARNINGS\*(C'\fR. This value is available |
| 583 |
since Encode version 2.99. |
| 584 |
.PP |
| 585 |
\fIFB_PERLQQ FB_HTMLCREF FB_XMLCREF\fR |
| 586 |
.IX Subsection "FB_PERLQQ FB_HTMLCREF FB_XMLCREF" |
| 587 |
.IP "perlqq mode (\fICHECK\fR = Encode::FB_PERLQQ)" 2 |
| 588 |
.IX Item "perlqq mode (CHECK = Encode::FB_PERLQQ)" |
| 589 |
.PD 0 |
| 590 |
.IP "HTML charref mode (\fICHECK\fR = Encode::FB_HTMLCREF)" 2 |
| 591 |
.IX Item "HTML charref mode (CHECK = Encode::FB_HTMLCREF)" |
| 592 |
.IP "XML charref mode (\fICHECK\fR = Encode::FB_XMLCREF)" 2 |
| 593 |
.IX Item "XML charref mode (CHECK = Encode::FB_XMLCREF)" |
| 594 |
.PD |
| 595 |
.PP |
| 596 |
For encodings that are implemented by the \f(CW\*(C`Encode::XS\*(C'\fR module, \f(CW\*(C`CHECK\*(C'\fR \f(CW\*(C`==\*(C'\fR |
| 597 |
\&\f(CW\*(C`Encode::FB_PERLQQ\*(C'\fR puts \f(CW\*(C`encode\*(C'\fR and \f(CW\*(C`decode\*(C'\fR into \f(CW\*(C`perlqq\*(C'\fR fallback mode. |
| 598 |
.PP |
| 599 |
When you decode, \f(CW\*(C`\ex\fR\f(CIHH\fR\f(CW\*(C'\fR is inserted for a malformed character, where |
| 600 |
\&\fIHH\fR is the hex representation of the octet that could not be decoded to |
| 601 |
utf8. When you encode, \f(CW\*(C`\ex{\fR\f(CIHHHH\fR\f(CW}\*(C'\fR will be inserted, where \fIHHHH\fR is |
| 602 |
the Unicode code point (in any number of hex digits) of the character that |
| 603 |
cannot be found in the character repertoire of the encoding. |
| 604 |
.PP |
| 605 |
The HTML/XML character reference modes are about the same. In place of |
| 606 |
\&\f(CW\*(C`\ex{\fR\f(CIHHHH\fR\f(CW}\*(C'\fR, HTML uses \f(CW\*(C`&#\fR\f(CINNN\fR\f(CW;\*(C'\fR where \fINNN\fR is a decimal number, and |
| 607 |
XML uses \f(CW\*(C`&#x\fR\f(CIHHHH\fR\f(CW;\*(C'\fR where \fIHHHH\fR is the hexadecimal number. |
| 608 |
.PP |
| 609 |
In \f(CW\*(C`Encode\*(C'\fR 2.10 or later, \f(CW\*(C`LEAVE_SRC\*(C'\fR is also implied. |
| 610 |
.PP |
| 611 |
\fIThe bitmask\fR |
| 612 |
.IX Subsection "The bitmask" |
| 613 |
.PP |
| 614 |
These modes are all actually set via a bitmask. Here is how the \f(CW\*(C`FB_\fR\f(CIXXX\fR\f(CW\*(C'\fR |
| 615 |
constants are laid out. You can import the \f(CW\*(C`FB_\fR\f(CIXXX\fR\f(CW\*(C'\fR constants via |
| 616 |
\&\f(CW\*(C`use Encode qw(:fallbacks)\*(C'\fR, and you can import the generic bitmask |
| 617 |
constants via \f(CW\*(C`use Encode qw(:fallback_all)\*(C'\fR. |
| 618 |
.PP |
| 619 |
.Vb 8 |
| 620 |
\& FB_DEFAULT FB_CROAK FB_QUIET FB_WARN FB_PERLQQ |
| 621 |
\& DIE_ON_ERR 0x0001 X |
| 622 |
\& WARN_ON_ERR 0x0002 X |
| 623 |
\& RETURN_ON_ERR 0x0004 X X |
| 624 |
\& LEAVE_SRC 0x0008 X |
| 625 |
\& PERLQQ 0x0100 X |
| 626 |
\& HTMLCREF 0x0200 |
| 627 |
\& XMLCREF 0x0400 |
| 628 |
.Ve |
| 629 |
.PP |
| 630 |
\fILEAVE_SRC\fR |
| 631 |
.IX Subsection "LEAVE_SRC" |
| 632 |
.PP |
| 633 |
.Vb 1 |
| 634 |
\& Encode::LEAVE_SRC |
| 635 |
.Ve |
| 636 |
.PP |
| 637 |
If the \f(CW\*(C`Encode::LEAVE_SRC\*(C'\fR bit is \fInot\fR set but \fICHECK\fR is set, then the |
| 638 |
source string to \fBencode()\fR or \fBdecode()\fR will be overwritten in place. |
| 639 |
If you're not interested in this, then bitwise-OR it with the bitmask. |
| 640 |
.SS "coderef for CHECK" |
| 641 |
.IX Subsection "coderef for CHECK" |
| 642 |
As of \f(CW\*(C`Encode\*(C'\fR 2.12, \f(CW\*(C`CHECK\*(C'\fR can also be a code reference which takes the |
| 643 |
ordinal value of the unmapped character as an argument and returns |
| 644 |
octets that represent the fallback character. For instance: |
| 645 |
.PP |
| 646 |
.Vb 1 |
| 647 |
\& $ascii = encode("ascii", $utf8, sub{ sprintf "<U+%04X>", shift }); |
| 648 |
.Ve |
| 649 |
.PP |
| 650 |
Acts like \f(CW\*(C`FB_PERLQQ\*(C'\fR but U+\fIXXXX\fR is used instead of \f(CW\*(C`\ex{\fR\f(CIXXXX\fR\f(CW}\*(C'\fR. |
| 651 |
.PP |
| 652 |
Fallback for \f(CW\*(C`decode\*(C'\fR must return decoded string (sequence of characters) |
| 653 |
and takes a list of ordinal values as its arguments. So for |
| 654 |
example if you wish to decode octets as UTF\-8, and use ISO\-8859\-15 as |
| 655 |
a fallback for bytes that are not valid UTF\-8, you could write |
| 656 |
.PP |
| 657 |
.Vb 4 |
| 658 |
\& $str = decode \*(AqUTF\-8\*(Aq, $octets, sub { |
| 659 |
\& my $tmp = join \*(Aq\*(Aq, map chr, @_; |
| 660 |
\& return decode \*(AqISO\-8859\-15\*(Aq, $tmp; |
| 661 |
\& }; |
| 662 |
.Ve |
| 663 |
.SH "Defining Encodings" |
| 664 |
.IX Header "Defining Encodings" |
| 665 |
To define a new encoding, use: |
| 666 |
.PP |
| 667 |
.Vb 2 |
| 668 |
\& use Encode qw(define_encoding); |
| 669 |
\& define_encoding($object, CANONICAL_NAME [, alias...]); |
| 670 |
.Ve |
| 671 |
.PP |
| 672 |
\&\fICANONICAL_NAME\fR will be associated with \fR\f(CI$object\fR\fI\fR. The object |
| 673 |
should provide the interface described in Encode::Encoding. |
| 674 |
If more than two arguments are provided, additional |
| 675 |
arguments are considered aliases for \fI\fR\f(CI$object\fR\fI\fR. |
| 676 |
.PP |
| 677 |
See Encode::Encoding for details. |
| 678 |
.SH "The UTF8 flag" |
| 679 |
.IX Header "The UTF8 flag" |
| 680 |
Before the introduction of Unicode support in Perl, the \f(CW\*(C`eq\*(C'\fR operator |
| 681 |
just compared the strings represented by two scalars. Beginning with |
| 682 |
Perl 5.8, \f(CW\*(C`eq\*(C'\fR compares two strings with simultaneous consideration of |
| 683 |
\&\fIthe UTF8 flag\fR. To explain why we made it so, I quote from page 402 of |
| 684 |
\&\fIProgramming Perl, 3rd ed.\fR |
| 685 |
.IP "Goal #1:" 2 |
| 686 |
.IX Item "Goal #1:" |
| 687 |
Old byte-oriented programs should not spontaneously break on the old |
| 688 |
byte-oriented data they used to work on. |
| 689 |
.IP "Goal #2:" 2 |
| 690 |
.IX Item "Goal #2:" |
| 691 |
Old byte-oriented programs should magically start working on the new |
| 692 |
character-oriented data when appropriate. |
| 693 |
.IP "Goal #3:" 2 |
| 694 |
.IX Item "Goal #3:" |
| 695 |
Programs should run just as fast in the new character-oriented mode |
| 696 |
as in the old byte-oriented mode. |
| 697 |
.IP "Goal #4:" 2 |
| 698 |
.IX Item "Goal #4:" |
| 699 |
Perl should remain one language, rather than forking into a |
| 700 |
byte-oriented Perl and a character-oriented Perl. |
| 701 |
.PP |
| 702 |
When \fIProgramming Perl, 3rd ed.\fR was written, not even Perl 5.6.0 had been |
| 703 |
born yet, many features documented in the book remained unimplemented for a |
| 704 |
long time. Perl 5.8 corrected much of this, and the introduction of the |
| 705 |
UTF8 flag is one of them. You can think of there being two fundamentally |
| 706 |
different kinds of strings and string-operations in Perl: one a |
| 707 |
byte-oriented mode for when the internal UTF8 flag is off, and the other a |
| 708 |
character-oriented mode for when the internal UTF8 flag is on. |
| 709 |
.PP |
| 710 |
This UTF8 flag is not visible in Perl scripts, exactly for the same reason |
| 711 |
you cannot (or rather, you \fIdon't have to\fR) see whether a scalar contains |
| 712 |
a string, an integer, or a floating-point number. But you can still peek |
| 713 |
and poke these if you will. See the next section. |
| 714 |
.SS "Messing with Perl's Internals" |
| 715 |
.IX Subsection "Messing with Perl's Internals" |
| 716 |
The following API uses parts of Perl's internals in the current |
| 717 |
implementation. As such, they are efficient but may change in a future |
| 718 |
release. |
| 719 |
.PP |
| 720 |
\fIis_utf8\fR |
| 721 |
.IX Subsection "is_utf8" |
| 722 |
.PP |
| 723 |
.Vb 1 |
| 724 |
\& is_utf8(STRING [, CHECK]) |
| 725 |
.Ve |
| 726 |
.PP |
| 727 |
[INTERNAL] Tests whether the UTF8 flag is turned on in the \fISTRING\fR. |
| 728 |
If \fICHECK\fR is true, also checks whether \fISTRING\fR contains well-formed |
| 729 |
UTF\-8. Returns true if successful, false otherwise. |
| 730 |
.PP |
| 731 |
Typically only necessary for debugging and testing. Don't use this flag as |
| 732 |
a marker to distinguish character and binary data, that should be decided |
| 733 |
for each variable when you write your code. |
| 734 |
.PP |
| 735 |
\&\fBCAVEAT\fR: If \fISTRING\fR has UTF8 flag set, it does \fBNOT\fR mean that |
| 736 |
\&\fISTRING\fR is UTF\-8 encoded and vice-versa. |
| 737 |
.PP |
| 738 |
As of Perl 5.8.1, utf8 also has the \f(CW\*(C`utf8::is_utf8\*(C'\fR function. |
| 739 |
.PP |
| 740 |
\fI_utf8_on\fR |
| 741 |
.IX Subsection "_utf8_on" |
| 742 |
.PP |
| 743 |
.Vb 1 |
| 744 |
\& _utf8_on(STRING) |
| 745 |
.Ve |
| 746 |
.PP |
| 747 |
[INTERNAL] Turns the \fISTRING\fR's internal UTF8 flag \fBon\fR. The \fISTRING\fR |
| 748 |
is \fInot\fR checked for containing only well-formed UTF\-8. Do not use this |
| 749 |
unless you \fIknow with absolute certainty\fR that the STRING holds only |
| 750 |
well-formed UTF\-8. Returns the previous state of the UTF8 flag (so please |
| 751 |
don't treat the return value as indicating success or failure), or \f(CW\*(C`undef\*(C'\fR |
| 752 |
if \fISTRING\fR is not a string. |
| 753 |
.PP |
| 754 |
\&\fBNOTE\fR: For security reasons, this function does not work on tainted values. |
| 755 |
.PP |
| 756 |
\fI_utf8_off\fR |
| 757 |
.IX Subsection "_utf8_off" |
| 758 |
.PP |
| 759 |
.Vb 1 |
| 760 |
\& _utf8_off(STRING) |
| 761 |
.Ve |
| 762 |
.PP |
| 763 |
[INTERNAL] Turns the \fISTRING\fR's internal UTF8 flag \fBoff\fR. Do not use |
| 764 |
frivolously. Returns the previous state of the UTF8 flag, or \f(CW\*(C`undef\*(C'\fR if |
| 765 |
\&\fISTRING\fR is not a string. Do not treat the return value as indicative of |
| 766 |
success or failure, because that isn't what it means: it is only the |
| 767 |
previous setting. |
| 768 |
.PP |
| 769 |
\&\fBNOTE\fR: For security reasons, this function does not work on tainted values. |
| 770 |
.SH "UTF\-8 vs. utf8 vs. UTF8" |
| 771 |
.IX Header "UTF-8 vs. utf8 vs. UTF8" |
| 772 |
.Vb 3 |
| 773 |
\& ....We now view strings not as sequences of bytes, but as sequences |
| 774 |
\& of numbers in the range 0 .. 2**32\-1 (or in the case of 64\-bit |
| 775 |
\& computers, 0 .. 2**64\-1) \-\- Programming Perl, 3rd ed. |
| 776 |
.Ve |
| 777 |
.PP |
| 778 |
That has historically been Perl's notion of UTF\-8, as that is how UTF\-8 was |
| 779 |
first conceived by Ken Thompson when he invented it. However, thanks to |
| 780 |
later revisions to the applicable standards, official UTF\-8 is now rather |
| 781 |
stricter than that. For example, its range is much narrower (0 .. 0x10_FFFF |
| 782 |
to cover only 21 bits instead of 32 or 64 bits) and some sequences |
| 783 |
are not allowed, like those used in surrogate pairs, the 31 non-character |
| 784 |
code points 0xFDD0 .. 0xFDEF, the last two code points in \fIany\fR plane |
| 785 |
(0x\fIXX\fR_FFFE and 0x\fIXX\fR_FFFF), all non-shortest encodings, etc. |
| 786 |
.PP |
| 787 |
The former default in which Perl would always use a loose interpretation of |
| 788 |
UTF\-8 has now been overruled: |
| 789 |
.PP |
| 790 |
.Vb 5 |
| 791 |
\& From: Larry Wall <[email protected]> |
| 792 |
\& Date: December 04, 2004 11:51:58 JST |
| 793 |
\& To: perl\[email protected] |
| 794 |
\& Subject: Re: Make Encode.pm support the real UTF\-8 |
| 795 |
\& Message\-Id: <[email protected]> |
| 796 |
\& |
| 797 |
\& On Fri, Dec 03, 2004 at 10:12:12PM +0000, Tim Bunce wrote: |
| 798 |
\& : I\*(Aqve no problem with \*(Aqutf8\*(Aq being perl\*(Aqs unrestricted uft8 encoding, |
| 799 |
\& : but "UTF\-8" is the name of the standard and should give the |
| 800 |
\& : corresponding behaviour. |
| 801 |
\& |
| 802 |
\& For what it\*(Aqs worth, that\*(Aqs how I\*(Aqve always kept them straight in my |
| 803 |
\& head. |
| 804 |
\& |
| 805 |
\& Also for what it\*(Aqs worth, Perl 6 will mostly default to strict but |
| 806 |
\& make it easy to switch back to lax. |
| 807 |
\& |
| 808 |
\& Larry |
| 809 |
.Ve |
| 810 |
.PP |
| 811 |
Got that? As of Perl 5.8.7, \fB"UTF\-8"\fR means UTF\-8 in its current |
| 812 |
sense, which is conservative and strict and security-conscious, whereas |
| 813 |
\&\fB"utf8"\fR means UTF\-8 in its former sense, which was liberal and loose and |
| 814 |
lax. \f(CW\*(C`Encode\*(C'\fR version 2.10 or later thus groks this subtle but critically |
| 815 |
important distinction between \f(CW"UTF\-8"\fR and \f(CW"utf8"\fR. |
| 816 |
.PP |
| 817 |
.Vb 2 |
| 818 |
\& encode("utf8", "\ex{FFFF_FFFF}", 1); # okay |
| 819 |
\& encode("UTF\-8", "\ex{FFFF_FFFF}", 1); # croaks |
| 820 |
.Ve |
| 821 |
.PP |
| 822 |
This distinction is also important for decoding. In the following, |
| 823 |
\&\f(CW$s\fR stores character U+200000, which exceeds UTF\-8's allowed range. |
| 824 |
\&\f(CW$s\fR thus stores an invalid Unicode code point: |
| 825 |
.PP |
| 826 |
.Vb 1 |
| 827 |
\& $s = decode("utf8", "\exf8\ex88\ex80\ex80\ex80"); |
| 828 |
.Ve |
| 829 |
.PP |
| 830 |
\&\f(CW"UTF\-8"\fR, by contrast, will either coerce the input to something valid: |
| 831 |
.PP |
| 832 |
.Vb 1 |
| 833 |
\& $s = decode("UTF\-8", "\exf8\ex88\ex80\ex80\ex80"); # U+FFFD |
| 834 |
.Ve |
| 835 |
.PP |
| 836 |
\&.. or croak: |
| 837 |
.PP |
| 838 |
.Vb 1 |
| 839 |
\& decode("UTF\-8", "\exf8\ex88\ex80\ex80\ex80", FB_CROAK|LEAVE_SRC); |
| 840 |
.Ve |
| 841 |
.PP |
| 842 |
In the \f(CW\*(C`Encode\*(C'\fR module, \f(CW"UTF\-8"\fR is actually a canonical name for |
| 843 |
\&\f(CW"utf\-8\-strict"\fR. That hyphen between the \f(CW"UTF"\fR and the \f(CW"8"\fR is |
| 844 |
critical; without it, \f(CW\*(C`Encode\*(C'\fR goes "liberal" and (perhaps overly\-)permissive: |
| 845 |
.PP |
| 846 |
.Vb 4 |
| 847 |
\& find_encoding("UTF\-8")\->name # is \*(Aqutf\-8\-strict\*(Aq |
| 848 |
\& find_encoding("utf\-8")\->name # ditto. names are case insensitive |
| 849 |
\& find_encoding("utf_8")\->name # ditto. "_" are treated as "\-" |
| 850 |
\& find_encoding("UTF8")\->name # is \*(Aqutf8\*(Aq. |
| 851 |
.Ve |
| 852 |
.PP |
| 853 |
Perl's internal UTF8 flag is called "UTF8", without a hyphen. It indicates |
| 854 |
whether a string is internally encoded as "utf8", also without a hyphen. |
| 855 |
.SH "SEE ALSO" |
| 856 |
.IX Header "SEE ALSO" |
| 857 |
Encode::Encoding, |
| 858 |
Encode::Supported, |
| 859 |
Encode::PerlIO, |
| 860 |
encoding, |
| 861 |
perlebcdic, |
| 862 |
"open" in perlfunc, |
| 863 |
perlunicode, perluniintro, perlunifaq, perlunitut |
| 864 |
utf8, |
| 865 |
the Perl Unicode Mailing List <http://lists.perl.org/list/perl\-unicode.html> |
| 866 |
.SH MAINTAINER |
| 867 |
.IX Header "MAINTAINER" |
| 868 |
This project was originated by the late Nick Ing-Simmons and later |
| 869 |
maintained by Dan Kogai \fI<[email protected]>\fR. See AUTHORS |
| 870 |
for a full list of people involved. For any questions, send mail to |
| 871 |
\&\fI<perl\[email protected]>\fR so that we can all share. |
| 872 |
.PP |
| 873 |
While Dan Kogai retains the copyright as a maintainer, credit |
| 874 |
should go to all those involved. See AUTHORS for a list of those |
| 875 |
who submitted code to the project. |
| 876 |
.SH COPYRIGHT |
| 877 |
.IX Header "COPYRIGHT" |
| 878 |
Copyright 2002\-2014 Dan Kogai \fI<[email protected]>\fR. |
| 879 |
.PP |
| 880 |
This library is free software; you can redistribute it and/or modify |
| 881 |
it under the same terms as Perl itself. |