Debian Codegraph @

bin/fix_latin

raw
1 #!/usr/bin/perl
2 ##############################################################################
3 #
4 # Script: fix_latin
5 #
6 # Author: Grant McLean <[email protected]>
7 #
8 # Description:
9 #
10 # A transcoding filter which takes mixed ASCII, UTF8 and Latin-1 data input and
11 # produces ASCII/UTF8 output.
12 #
13 # For documentation see: ./fix_latin -?
14 #
15
16 use strict;
17 use warnings;
18
19 require 5.008;
20
21 use Pod::Usage;
22 use Getopt::Long qw(GetOptions);
23 use Encoding::FixLatin qw(fix_latin);
24
25 my %opt;
26
27 if(!GetOptions(\%opt, 'help|?', 'version|v', 'use_xs|use-xs=s')) {
28 pod2usage(-exitval => 1, -verbose => 0);
29 }
30
31 pod2usage(-exitstatus => 0, -verbose => 2) if($opt{'help'});
32
33 if($opt{version}) {
34 fix_latin(''); # Load XS layer if available
35 my $xs_version = defined($Encoding::FixLatin::XS::VERSION)
36 ? "version $Encoding::FixLatin::XS::VERSION"
37 : 'not installed';
38 print "Encoding::FixLatin version $Encoding::FixLatin::VERSION\n";
39 print "Encoding::FixLatin::XS $xs_version\n";
40 exit;
41 }
42
43 my @use_xs = ( use_xs => $opt{use_xs} || 'auto' );
44
45 while(<>) {
46 $_ = fix_latin($_, bytes_only => 1, @use_xs) if /[^\x00-\x7f]/;
47 print;
48 }
49
50 exit 0;
51
52 __END__
53
54 =head1 NAME
55
56 fix_latin - filters a data stream that is predominantly utf8 and 'fixes' any
57 latin (ie: non-ASCII 8 bit) characters
58
59 =head1 SYNOPSIS
60
61 fix_latin options <input_file >output_file
62
63 Options:
64
65 --use-xs <value> 'auto' | 'always' | 'never'
66 --version list version number
67 --help detailed help message
68
69 =head1 DESCRIPTION
70
71 The script acts as a filter, taking source data which may contain a mix of
72 ASCII, UTF8, ISO8859-1 and CP1252 characters, and producing output will be all
73 ASCII/UTF8.
74
75 Multi-byte UTF8 characters will be passed through unchanged (although over-long
76 UTF8 byte sequences will be converted to the shortest normal form). Single
77 byte characters will be converted as follows:
78
79 0x00 - 0x7F ASCII - passed through unchanged
80 0x80 - 0x9F Converted to UTF8 using CP1252 mappings
81 0xA0 - 0xFF Converted to UTF8 using Latin-1 mappings
82
83 =head1 OPTIONS
84
85 =over 4
86
87 =item B<< --use-xs 'auto' | 'always' | 'never' >>
88
89 Override default ('auto') behaviour of trying to use XS module and falling back
90 to pure-Perl version if not available. Set to 'never' to always use the Perl
91 version or 'always' to always use XS and die if not available.
92
93 =item B<--version> (alias -v)
94
95 Display version number of underlying Encoding::FixLatin and XS modules.
96
97 =item B<--help> (alias -?)
98
99 Display this documentation.
100
101 =back
102
103 =head1 EXAMPLES
104
105 This script was originally written to assist in converting a Postgres database
106 from SQL-ASCII encoding to UNICODE UTF8 encoding. The following examples
107 illustrate its use in that context.
108
109 If you have a SQL format dump file that you would normally restore by piping
110 into 'psql', you can simply filter the dump file through this script:
111
112 fix_latin < dump_file | psql -d database
113
114 If you have a compressed dump file that you would normally restore using
115 'pg_restore', you can omit the '-d' option on pg_restore and pipe the resulting
116 SQL through this script and into psql:
117
118 pg_restore -O dump_file | fix_latin | psql -d database
119
120 To take a look at non-ASCII lines in the dump file:
121
122 perl -ne '/^COPY (\S+)/ and $t = $1; print "$t:$_" if /[^\x00-\x7F]/' dump_file
123
124 =head1 SEE ALSO
125
126 This script is implemented using the Encoding::FixLatin Perl module. For more
127 details see the module documentation with the command:
128
129 perldoc Encoding::FixLatin
130
131 In particular you should read the 'LIMITATIONS' section to understand the
132 circumstances under which data corruption might occur.
133
134 =head1 COPYRIGHT & LICENSE
135
136 Copyright 2009-2014 Grant McLean C<< <[email protected]> >>
137
138 This program is free software; you can redistribute it and/or modify it
139 under the same terms as Perl itself.
140
141 =cut
142
143
144