| 1 |
#!/usr/bin/perl |
| 2 |
############################################################################## |
| 3 |
# |
| 4 |
# Script: fix_latin |
| 5 |
# |
| 6 |
# Author: Grant McLean <[email protected]> |
| 7 |
# |
| 8 |
# Description: |
| 9 |
# |
| 10 |
# A transcoding filter which takes mixed ASCII, UTF8 and Latin-1 data input and |
| 11 |
# produces ASCII/UTF8 output. |
| 12 |
# |
| 13 |
# For documentation see: ./fix_latin -? |
| 14 |
# |
| 15 |
|
| 16 |
use strict; |
| 17 |
use warnings; |
| 18 |
|
| 19 |
require 5.008; |
| 20 |
|
| 21 |
use Pod::Usage; |
| 22 |
use Getopt::Long qw(GetOptions); |
| 23 |
use Encoding::FixLatin qw(fix_latin); |
| 24 |
|
| 25 |
my %opt; |
| 26 |
|
| 27 |
if(!GetOptions(\%opt, 'help|?', 'version|v', 'use_xs|use-xs=s')) { |
| 28 |
pod2usage(-exitval => 1, -verbose => 0); |
| 29 |
} |
| 30 |
|
| 31 |
pod2usage(-exitstatus => 0, -verbose => 2) if($opt{'help'}); |
| 32 |
|
| 33 |
if($opt{version}) { |
| 34 |
fix_latin(''); # Load XS layer if available |
| 35 |
my $xs_version = defined($Encoding::FixLatin::XS::VERSION) |
| 36 |
? "version $Encoding::FixLatin::XS::VERSION" |
| 37 |
: 'not installed'; |
| 38 |
print "Encoding::FixLatin version $Encoding::FixLatin::VERSION\n"; |
| 39 |
print "Encoding::FixLatin::XS $xs_version\n"; |
| 40 |
exit; |
| 41 |
} |
| 42 |
|
| 43 |
my @use_xs = ( use_xs => $opt{use_xs} || 'auto' ); |
| 44 |
|
| 45 |
while(<>) { |
| 46 |
$_ = fix_latin($_, bytes_only => 1, @use_xs) if /[^\x00-\x7f]/; |
| 47 |
print; |
| 48 |
} |
| 49 |
|
| 50 |
exit 0; |
| 51 |
|
| 52 |
__END__ |
| 53 |
|
| 54 |
=head1 NAME |
| 55 |
|
| 56 |
fix_latin - filters a data stream that is predominantly utf8 and 'fixes' any |
| 57 |
latin (ie: non-ASCII 8 bit) characters |
| 58 |
|
| 59 |
=head1 SYNOPSIS |
| 60 |
|
| 61 |
fix_latin options <input_file >output_file |
| 62 |
|
| 63 |
Options: |
| 64 |
|
| 65 |
--use-xs <value> 'auto' | 'always' | 'never' |
| 66 |
--version list version number |
| 67 |
--help detailed help message |
| 68 |
|
| 69 |
=head1 DESCRIPTION |
| 70 |
|
| 71 |
The script acts as a filter, taking source data which may contain a mix of |
| 72 |
ASCII, UTF8, ISO8859-1 and CP1252 characters, and producing output will be all |
| 73 |
ASCII/UTF8. |
| 74 |
|
| 75 |
Multi-byte UTF8 characters will be passed through unchanged (although over-long |
| 76 |
UTF8 byte sequences will be converted to the shortest normal form). Single |
| 77 |
byte characters will be converted as follows: |
| 78 |
|
| 79 |
0x00 - 0x7F ASCII - passed through unchanged |
| 80 |
0x80 - 0x9F Converted to UTF8 using CP1252 mappings |
| 81 |
0xA0 - 0xFF Converted to UTF8 using Latin-1 mappings |
| 82 |
|
| 83 |
=head1 OPTIONS |
| 84 |
|
| 85 |
=over 4 |
| 86 |
|
| 87 |
=item B<< --use-xs 'auto' | 'always' | 'never' >> |
| 88 |
|
| 89 |
Override default ('auto') behaviour of trying to use XS module and falling back |
| 90 |
to pure-Perl version if not available. Set to 'never' to always use the Perl |
| 91 |
version or 'always' to always use XS and die if not available. |
| 92 |
|
| 93 |
=item B<--version> (alias -v) |
| 94 |
|
| 95 |
Display version number of underlying Encoding::FixLatin and XS modules. |
| 96 |
|
| 97 |
=item B<--help> (alias -?) |
| 98 |
|
| 99 |
Display this documentation. |
| 100 |
|
| 101 |
=back |
| 102 |
|
| 103 |
=head1 EXAMPLES |
| 104 |
|
| 105 |
This script was originally written to assist in converting a Postgres database |
| 106 |
from SQL-ASCII encoding to UNICODE UTF8 encoding. The following examples |
| 107 |
illustrate its use in that context. |
| 108 |
|
| 109 |
If you have a SQL format dump file that you would normally restore by piping |
| 110 |
into 'psql', you can simply filter the dump file through this script: |
| 111 |
|
| 112 |
fix_latin < dump_file | psql -d database |
| 113 |
|
| 114 |
If you have a compressed dump file that you would normally restore using |
| 115 |
'pg_restore', you can omit the '-d' option on pg_restore and pipe the resulting |
| 116 |
SQL through this script and into psql: |
| 117 |
|
| 118 |
pg_restore -O dump_file | fix_latin | psql -d database |
| 119 |
|
| 120 |
To take a look at non-ASCII lines in the dump file: |
| 121 |
|
| 122 |
perl -ne '/^COPY (\S+)/ and $t = $1; print "$t:$_" if /[^\x00-\x7F]/' dump_file |
| 123 |
|
| 124 |
=head1 SEE ALSO |
| 125 |
|
| 126 |
This script is implemented using the Encoding::FixLatin Perl module. For more |
| 127 |
details see the module documentation with the command: |
| 128 |
|
| 129 |
perldoc Encoding::FixLatin |
| 130 |
|
| 131 |
In particular you should read the 'LIMITATIONS' section to understand the |
| 132 |
circumstances under which data corruption might occur. |
| 133 |
|
| 134 |
=head1 COPYRIGHT & LICENSE |
| 135 |
|
| 136 |
Copyright 2009-2014 Grant McLean C<< <[email protected]> >> |
| 137 |
|
| 138 |
This program is free software; you can redistribute it and/or modify it |
| 139 |
under the same terms as Perl itself. |
| 140 |
|
| 141 |
=cut |
| 142 |
|
| 143 |
|
| 144 |
|