| 1 |
NAME |
| 2 |
File::BOM - Utilities for handling Byte Order Marks |
| 3 |
|
| 4 |
SYNOPSIS |
| 5 |
use File::BOM qw( :all ) |
| 6 |
|
| 7 |
high-level functions |
| 8 |
# read a file with encoding from the BOM: |
| 9 |
open_bom(FH, $file) |
| 10 |
open_bom(FH, $file, ':utf8') # the same but with a default encoding |
| 11 |
|
| 12 |
# get encoding too |
| 13 |
$encoding = open_bom(FH, $file, ':utf8'); |
| 14 |
|
| 15 |
# open a potentially unseekable file: |
| 16 |
($encoding, $spillage) = open_bom(FH, $file, ':utf8'); |
| 17 |
|
| 18 |
# change encoding of an open handle according to BOM |
| 19 |
$encoding = defuse(*HANDLE); |
| 20 |
($encoding, $spillage) = defuse(*HANDLE); |
| 21 |
|
| 22 |
# Decode a string according to leading BOM: |
| 23 |
$unicode = decode_from_bom($string_with_bom); |
| 24 |
|
| 25 |
# Decode a string and get the encoding: |
| 26 |
($unicode, $encoding) = decode_from_bom($string_with_bom) |
| 27 |
|
| 28 |
PerlIO::via interface |
| 29 |
# Read the Right Thing from a unicode file with BOM: |
| 30 |
open(HANDLE, '<:via(File::BOM)', $filename) |
| 31 |
|
| 32 |
# Writing little-endian UTF-16 file with BOM: |
| 33 |
open(HANDLE, '>:encoding(UTF-16LE):via(File::BOM)', $filename) |
| 34 |
|
| 35 |
lower-level functions |
| 36 |
# read BOM encoding from a filehandle: |
| 37 |
$encoding = get_encoding_from_filehandle(FH) |
| 38 |
|
| 39 |
# Get encoding even if FH is unseekable: |
| 40 |
($encoding, $spillage) = get_encoding_from_filehandle(FH); |
| 41 |
|
| 42 |
# Get encoding from a known unseekable handle: |
| 43 |
($encdoing, $spillage) = get_encoding_from_stream(FH); |
| 44 |
|
| 45 |
# get encoding and BOM length from BOM at start of string: |
| 46 |
($encoding, $offset) = get_encoding_from_bom($string); |
| 47 |
|
| 48 |
variables |
| 49 |
# print a BOM for a known encoding |
| 50 |
print FH $enc2bom{$encoding}; |
| 51 |
|
| 52 |
# get an encoding from a known BOM |
| 53 |
$enc = $bom2enc{$bom} |
| 54 |
|
| 55 |
DESCRIPTION |
| 56 |
This module provides functions for handling unicode byte order marks, |
| 57 |
which are to be found at the beginning of some files and streams. |
| 58 |
|
| 59 |
For details about what a byte order mark is, see |
| 60 |
<http://www.unicode.org/unicode/faq/utf_bom.html#BOM> |
| 61 |
|
| 62 |
The intention of File::BOM is for files with BOMs to be readable as |
| 63 |
seamlessly as possible, regardless of the encoding used. To that end, |
| 64 |
several different interfaces are available, as shown in the synopsis |
| 65 |
above. |
| 66 |
|
| 67 |
EXPORTS |
| 68 |
Nothing by default. |
| 69 |
|
| 70 |
symbols |
| 71 |
* open_bom() |
| 72 |
|
| 73 |
* defuse() |
| 74 |
|
| 75 |
* decode_from_bom() |
| 76 |
|
| 77 |
* get_encoding_from_filehandle() |
| 78 |
|
| 79 |
* get_encoding_from_stream() |
| 80 |
|
| 81 |
* get_encoding_from_bom() |
| 82 |
|
| 83 |
* %bom2enc |
| 84 |
|
| 85 |
* %enc2bom |
| 86 |
|
| 87 |
tags |
| 88 |
* :all |
| 89 |
|
| 90 |
All of the above |
| 91 |
|
| 92 |
* :subs |
| 93 |
|
| 94 |
subroutines only |
| 95 |
|
| 96 |
* :vars |
| 97 |
|
| 98 |
just %bom2enc and %enc2bom |
| 99 |
|
| 100 |
VARIABLES |
| 101 |
%bom2enc |
| 102 |
Maps Byte Order marks to their encodings. |
| 103 |
|
| 104 |
The keys of this hash are strings which represent the BOMs, the values |
| 105 |
are their encodings, in a format which is understood by Encode |
| 106 |
|
| 107 |
The encodings represented in this hash are: UTF-8, UTF-16BE, UTF-16LE, |
| 108 |
UTF-32BE and UTF-32LE |
| 109 |
|
| 110 |
%enc2bom |
| 111 |
A reverse-lookup hash for bom2enc, with a few aliases used in Encode, |
| 112 |
namely utf8, iso-10646-1 and UCS-2. |
| 113 |
|
| 114 |
Note that UTF-16, UTF-32 and UCS-4 are not included in this hash. Mainly |
| 115 |
because Encode::encode automatically puts BOMs on output. See |
| 116 |
Encode::Unicode |
| 117 |
|
| 118 |
FUNCTIONS |
| 119 |
open_bom |
| 120 |
$encoding = open_bom(HANDLE, $filename, $default_mode) |
| 121 |
|
| 122 |
($encoding, $spill) = open_bom(HANDLE, $filename, $default_mode) |
| 123 |
|
| 124 |
opens HANDLE for reading on $filename, setting the mode to the |
| 125 |
appropriate encoding for the BOM stored in the file. |
| 126 |
|
| 127 |
On failure, a fatal error is raised, see the DIAGNOSTICS section for |
| 128 |
details on how to catch these. This is in order to allow the return |
| 129 |
value(s) to be used for other purposes. |
| 130 |
|
| 131 |
If the file doesn't contain a BOM, $default_mode is used instead. Hence: |
| 132 |
|
| 133 |
open_bom(FH, 'my_file.txt', ':utf8') |
| 134 |
|
| 135 |
Opens my_file.txt for reading in an appropriate encoding found from the |
| 136 |
BOM in that file, or as a UTF-8 file if none is found. |
| 137 |
|
| 138 |
In the absence of a $default_mode argument, the following 2 calls should |
| 139 |
be equivalent: |
| 140 |
|
| 141 |
open_bom(FH, 'no_bom.txt'); |
| 142 |
|
| 143 |
open(FH, '<', 'no_bom.txt'); |
| 144 |
|
| 145 |
If an undefined value is passed as the handle, a symbol will be |
| 146 |
generated for it like open() does: |
| 147 |
|
| 148 |
# create filehandle on the fly |
| 149 |
$enc = open_bom(my $fh, $filename, ':utf8'); |
| 150 |
$line = <$fh>; |
| 151 |
|
| 152 |
The filehandle will be cued up to read after the BOM. Unseekable files |
| 153 |
(e.g. fifos) will cause croaking, unless called in list context to catch |
| 154 |
spillage from the handle. Any spillage will be automatically decoded |
| 155 |
from the encoding, if found. |
| 156 |
|
| 157 |
e.g. |
| 158 |
|
| 159 |
# croak if my_socket is unseekable |
| 160 |
open_bom(FH, 'my_socket'); |
| 161 |
|
| 162 |
# keep spillage if my_socket is unseekable |
| 163 |
($encoding, $spillage) = open_bom(FH, 'my_socket'); |
| 164 |
|
| 165 |
# discard any spillage from open_bom |
| 166 |
($encoding) = open_bom(FH, 'my_socket'); |
| 167 |
|
| 168 |
defuse |
| 169 |
$enc = defuse(FH); |
| 170 |
|
| 171 |
($enc, $spill) = defuse(FH); |
| 172 |
|
| 173 |
FH should be a filehandle opened for reading, it will have the relevant |
| 174 |
encoding layer pushed onto it be binmode if a BOM is found. Spillage |
| 175 |
should be Unicode, not bytes. |
| 176 |
|
| 177 |
Any uncaptured spillage will be silently lost. If the handle is |
| 178 |
unseekable, use list context to avoid data loss. |
| 179 |
|
| 180 |
If no BOM is found, the mode will be unaffected. |
| 181 |
|
| 182 |
decode_from_bom |
| 183 |
$unicode_string = decode_from_bom($string, $default, $check) |
| 184 |
|
| 185 |
($unicode_string, $encoding) = decode_from_bom($string, $default, $check) |
| 186 |
|
| 187 |
Reads a BOM from the beginning of $string, decodes $string (minus the |
| 188 |
BOM) and returns it to you as a perl unicode string. |
| 189 |
|
| 190 |
if $string doesn't have a BOM, $default is used instead. |
| 191 |
|
| 192 |
$check, if supplied, is passed to Encode::decode as the third argument. |
| 193 |
|
| 194 |
If there's no BOM and no default, the original string is returned and |
| 195 |
encoding is ''. |
| 196 |
|
| 197 |
See Encode |
| 198 |
|
| 199 |
get_encoding_from_filehandle |
| 200 |
$encoding = get_encoding_from_filehandle(HANDLE) |
| 201 |
|
| 202 |
($encoding, $spillage) = get_encoding_from_filehandle(HANDLE) |
| 203 |
|
| 204 |
Returns the encoding found in the given filehandle. |
| 205 |
|
| 206 |
The handle should be opened in a non-unicode way (e.g. mode '<:bytes') |
| 207 |
so that the BOM can be read in its natural state. |
| 208 |
|
| 209 |
After calling, the handle will be set to read at a point after the BOM |
| 210 |
(or at the beginning of the file if no BOM was found) |
| 211 |
|
| 212 |
If called in scalar context, unseekable handles cause a croak(). |
| 213 |
|
| 214 |
If called in list context, unseekable handles will be read byte-by-byte |
| 215 |
and any spillage will be returned. See get_encoding_from_stream() |
| 216 |
|
| 217 |
get_encoding_from_stream |
| 218 |
($encoding, $spillage) = get_encoding_from_stream(*FH); |
| 219 |
|
| 220 |
Read a BOM from an unrewindable source. This means reading the stream |
| 221 |
one byte at a time until either a BOM is found or every possible BOM is |
| 222 |
ruled out. Any non-BOM bytes read from the handle will be returned in |
| 223 |
$spillage. |
| 224 |
|
| 225 |
If a BOM is found and the spillage contains a partial character (judging |
| 226 |
by the expected character width for the encoding) more bytes will be |
| 227 |
read from the handle to ensure that a complete character is returned. |
| 228 |
|
| 229 |
Spillage is always in bytes, not characters. |
| 230 |
|
| 231 |
This function is less efficient than get_encoding_from_filehandle, but |
| 232 |
should work just as well on a seekable handle as on an unseekable one. |
| 233 |
|
| 234 |
get_encoding_from_bom |
| 235 |
($encoding, $offset) = get_encoding_from_bom($string) |
| 236 |
|
| 237 |
Returns the encoding and length in bytes of the BOM in $string. |
| 238 |
|
| 239 |
If there is no BOM, an empty string is returned and $offset is zero. |
| 240 |
|
| 241 |
To get the data from the string, the following should work: |
| 242 |
|
| 243 |
use Encode; |
| 244 |
|
| 245 |
my($encoding, $offset) = get_encoding_from_bom($string); |
| 246 |
|
| 247 |
if ($encoding) { |
| 248 |
$string = decode($encoding, substr($string, $offset)) |
| 249 |
} |
| 250 |
|
| 251 |
PerlIO::via interface |
| 252 |
File::BOM can be used as a PerlIO::via interface. |
| 253 |
|
| 254 |
open(HANDLE, '<:via(File::BOM)', 'my_file.txt'); |
| 255 |
|
| 256 |
open(HANDLE, '>:encoding(UTF-16LE):via(File::BOM)', 'out_file.txt'); |
| 257 |
print "foo\n"; # BOM is written to file here |
| 258 |
|
| 259 |
This method is less prone to errors on non-seekable files as spillage is |
| 260 |
incorporated into an internal buffer, but it doesn't give you any |
| 261 |
information about the encoding being used, or indeed whether or not a |
| 262 |
BOM was present. |
| 263 |
|
| 264 |
There are a few known problems with this interface, especially |
| 265 |
surrounding seek() and tell(), please see the BUGS section for more |
| 266 |
details about this. |
| 267 |
|
| 268 |
Reading |
| 269 |
The via(File::BOM) layer must be added before the handle is read from, |
| 270 |
otherwise any BOM will be missed. If there is no BOM, no decoding will |
| 271 |
be done. |
| 272 |
|
| 273 |
Because of a limitation in PerlIO::via, read() always works on bytes, |
| 274 |
not characters. BOM decoding will still be done but output will be bytes |
| 275 |
of UTF-8. |
| 276 |
|
| 277 |
open(BOM, '<:via(File::BOM)', $file); |
| 278 |
$bytes_read = read(BOM, $buffer, $length); |
| 279 |
$unicode = decode('UTF-8', $buffer, Encode::FB_QUIET); |
| 280 |
|
| 281 |
# Now $unicode is valid unicode and $buffer contains any left-over bytes |
| 282 |
|
| 283 |
Writing |
| 284 |
Add the via(File::BOM) layer on top of a unicode encoding layer to print |
| 285 |
a BOM at the start of the output file. This needs to be done before any |
| 286 |
data is written. The BOM is written as part of the first print command |
| 287 |
on the handle, so if you don't print anything to the handle, you won't |
| 288 |
get a BOM. |
| 289 |
|
| 290 |
There is a "Wide character in print" warning generated when the |
| 291 |
via(File::BOM) layer doesn't receive utf8 on writing. This glitch was |
| 292 |
resolved in perl version 5.8.7, but if your perl version is older than |
| 293 |
that, you'll need to make sure that the via(File::BOM) layer receives |
| 294 |
utf8 like this: |
| 295 |
|
| 296 |
# This works OK |
| 297 |
open(FH, '>:encoding(UTF-16LE):via(File::BOM):utf8', $filename) |
| 298 |
|
| 299 |
# This generates warnings with older perls |
| 300 |
open(FH, '>:encoding(UTF-16LE):via(File::BOM)', $filename) |
| 301 |
|
| 302 |
Seeking |
| 303 |
Seeking with SEEK_SET results in an offset equal to the length of any |
| 304 |
detected BOM being applied to the position parameter. Thus: |
| 305 |
|
| 306 |
# Seek to end of BOM (not start of file!) |
| 307 |
seek(FILE_BOM_HANDLE, 0, SEEK_SET) |
| 308 |
|
| 309 |
Telling |
| 310 |
In order to work correctly with seek(), tell() also returns a postion |
| 311 |
adjusted by the length of the BOM. |
| 312 |
|
| 313 |
SEE ALSO |
| 314 |
* Encode |
| 315 |
|
| 316 |
* Encode::Unicode |
| 317 |
|
| 318 |
* <http://www.unicode.org/unicode/faq/utf_bom.html#BOM> |
| 319 |
|
| 320 |
DIAGNOSTICS |
| 321 |
The following exceptions are raised via croak() |
| 322 |
|
| 323 |
* Couldn't read '<filename>': $! |
| 324 |
|
| 325 |
open_bom() couldn't open the given file for reading |
| 326 |
|
| 327 |
* Couldn't set binmode of handle opened on '<filename>' to '<mode>': |
| 328 |
$! |
| 329 |
|
| 330 |
open_bom() couldn't set the binmode of the handle |
| 331 |
|
| 332 |
* No string |
| 333 |
|
| 334 |
decode_from_bom called on an undefined value |
| 335 |
|
| 336 |
* Unseekable handle: $! |
| 337 |
|
| 338 |
get_encoding_from_filehandle() or open_bom() called on an unseekable |
| 339 |
file or handle in scalar context. |
| 340 |
|
| 341 |
* Couldn't read from handle: $! |
| 342 |
|
| 343 |
_get_encoding_seekable() couldn't read the handle. This function is |
| 344 |
called from get_encoding_from_filehandle(), defuse() and open_bom() |
| 345 |
|
| 346 |
* Couldn't reset read position: $! |
| 347 |
|
| 348 |
_get_encoding_seekable couldn't seek to the position after the BOM. |
| 349 |
|
| 350 |
* Couldn't read byte: $! |
| 351 |
|
| 352 |
get_encoding_from_stream couldn't read from the handle. This |
| 353 |
function is called from get_encoding_from_filehandle() and |
| 354 |
open_bom() when the handle or file is unseekable. |
| 355 |
|
| 356 |
BUGS |
| 357 |
Older versions of PerlIO::via have a few problems with writing, see |
| 358 |
above. |
| 359 |
|
| 360 |
The current version of PerlIO::via has limitations with regard to seek |
| 361 |
and tell, currently only line-wise seek and tell are supported by this |
| 362 |
module. If read() is used to read partial lines, tell() will still give |
| 363 |
the position of the end of the last line read. |
| 364 |
|
| 365 |
Under windows, tell() seems to return erroneously when reading files |
| 366 |
with unix line endings. |
| 367 |
|
| 368 |
Under windows, warnings may be generated when using the PerlIO::via |
| 369 |
interface to read UTF-16LE and UTF-32LE encoded files. This seems to be |
| 370 |
a bug in the relevant encoding(...) layers. |
| 371 |
|
| 372 |
AUTHOR |
| 373 |
Matt Lawrence <[email protected]> |
| 374 |
|
| 375 |
With thanks to Mark Fowler and Steve Purkis for additional tests and |
| 376 |
advice. |
| 377 |
|
| 378 |
COPYRIGHT |
| 379 |
Copyright 2005 Matt Lawrence, All Rights Reserved. |
| 380 |
|
| 381 |
This program is free software; you can redistribute it and/or modify it |
| 382 |
under the same terms as Perl itself. |
| 383 |
|