Debian Codegraph @

bin/tabmerge

raw
1 #!perl
2
3 use strict;
4 use warnings;
5 use version;
6 use English qw( -no_match_vars );
7 use File::Basename;
8 use Getopt::Long;
9 use Pod::Usage;
10 use Readonly;
11 use Text::RecordParser;
12 use Text::TabularDisplay;
13
14 Readonly our $VERSION => 1.12;
15 Readonly my $COMMA => q{,};
16 Readonly my $COMMA_SPACE => q{, };
17 Readonly my $DASH => q{-};
18 Readonly my $EMPTY_STR => q{};
19 Readonly my $TAB => qq{\t};
20 Readonly my $NEWLINE => qq{\n};
21
22 my $fs = $TAB;
23 my $rs = $NEWLINE;
24 my ( $fields, $min, $max, $list, $sort, $stdout,
25 $help, $man_page, $show_version );
26
27 GetOptions(
28 'f|fields:s' => \$fields,
29 'help' => \$help,
30 'fs:s' => \$fs,
31 'l|list' => \$list,
32 'man' => \$man_page,
33 'min' => \$min,
34 'max' => \$max,
35 'rs:s' => \$rs,
36 's|sort:s' => \$sort,
37 'stdout' => \$stdout,
38 'version' => \$show_version,
39 ) or pod2usage;
40
41 if ( $help || $man_page ) {
42 pod2usage({
43 -exitval => 0,
44 -verbose => $man_page ? 2 : 1
45 });
46 };
47
48 if ( $show_version ) {
49 my $prog = basename( $PROGRAM_NAME );
50 print "$prog $VERSION\n";
51 exit 0;
52 }
53
54 if ( scalar @ARGV < 2 ) {
55 pod2usage('Please supply two or more files');
56 }
57
58 my @files = @ARGV or pod2usage('No input files');
59
60 $min = 1 unless $max or $fields;
61
62 my $p = Text::RecordParser->new(
63 field_separator => $fs,
64 record_separator => $rs,
65 );
66
67 if ( $list ) {
68 list( $p, @files );
69 }
70 elsif ( $min || $max || $fields ) {
71 intersection(
72 fields => $fields,
73 files => \@files,
74 parser => $p,
75 sort => $sort,
76 type => $min ? 'min' : 'max',
77 );
78 }
79 else {
80 pod2usage(q[Can't figure out what to do.]);
81 }
82
83 # -------------------------------------------------------------------
84 sub intersection {
85 my %args = @_;
86 my $p = $args{'parser'};
87 my $type = $args{'type'};
88 my $files = $args{'files'};
89 my $fields = $args{'fields'} || $EMPTY_STR; # optional
90 my $sort = $args{'sort'} || $EMPTY_STR; # optional
91
92 my %fields;
93 for my $file ( @$files ) {
94 $p->filename( $file );
95 $p->bind_header;
96 $fields{ $_ }++ for $p->field_list;
97 }
98
99 my @intersect;
100 if ( $fields ) {
101 @intersect = map { s/^\s+|\s+$//g; $_ } split /$COMMA/, $fields;
102 die_if_bad( \%fields, \@intersect );
103 }
104 else {
105 my $no_files = scalar @$files;
106 for my $fld ( keys %fields ) {
107 push @intersect, $fld if
108 ( $type eq 'max' ) ||
109 ( $type eq 'min' && $fields{ $fld } == $no_files )
110 ;
111 }
112
113 @intersect = sort @intersect;
114 }
115
116 die "No intersection!\n" unless @intersect;
117
118 my @data;
119 for my $file ( @$files ) {
120 $p->filename( $file );
121 $p->bind_header;
122 while ( my $rec = $p->fetchrow_hashref ) {
123 push @data, { map { $_, $rec->{ $_ } } @intersect };
124 }
125 }
126
127 if ( my @sort = map { s/^\s+|\s+$//g; $_ } split /$COMMA/, $sort ) {
128 die_if_bad( \%fields, \@sort );
129
130 @data =
131 map { $_->[1] }
132 sort { $a->[0] cmp $b->[0] }
133 map { [ join($DASH, @{ $_ }{ @sort } ), $_ ] }
134 @data;
135 }
136
137 if ( $stdout ) {
138 print join( $fs, @intersect ), $rs;
139 for my $rec ( @data ) {
140 print join( $fs,
141 map { defined $rec->{$_} ? $rec->{$_} : $EMPTY_STR } @intersect
142 ), $rs;
143 }
144 }
145 else {
146 my $tab = Text::TabularDisplay->new( @intersect );
147 for my $rec ( @data ) {
148 $tab->add( map { $rec->{ $_ } } @intersect );
149 }
150 print $tab->render, $NEWLINE;
151 }
152 }
153
154 # -------------------------------------------------------------------
155 sub list {
156 my ( $p, @files ) = @_;
157
158 my %fields;
159 for my $file ( @files ) {
160 $p->filename( $file );
161 $p->bind_header;
162 $fields{ $_ }++ for $p->field_list;
163 }
164
165 my $tab = Text::TabularDisplay->new('Field', 'No. Times Present');
166 $tab->add( $_, $fields{ $_ } ) for sort keys %fields;
167 print $tab->render, $NEWLINE;
168 }
169
170 # -------------------------------------------------------------------
171 sub die_if_bad {
172 my ( $fields, $check ) = @_;
173
174 my @bad;
175 for my $fld ( @$check ) {
176 push @bad, $fld unless $fields->{ $fld };
177 }
178
179 if ( @bad ) {
180 die sprintf "Bad field name%s: %s$NEWLINE",
181 scalar @bad > 1 ? 's' : $EMPTY_STR,
182 join $COMMA_SPACE, @bad
183 ;
184 }
185 else {
186 return 1;
187 }
188 }
189
190 __END__
191
192 # -------------------------------------------------------------------
193
194 # $Id: tabmerge,v 1.11 2006/03/07 17:20:00 kclark Exp $
195
196 =pod
197
198 =head1 NAME
199
200 tabmerge - unify delimited files on common fields
201
202 =head1 SYNOPSIS
203
204 tabmerge [action] [options] file1 file2 [...]
205
206 Actions:
207
208 --min Take only fields present in all files [DEFAULT]
209 --max Take all fields present
210 -f|--fields=f1[,f2] Take only the fields mentioned in the
211 comma-separated list
212
213 Options:
214
215 -l|--list List available fields
216 --fs=x Use "x" as the field separator
217 (default is tab "\t")
218 --rs=x Use "x" as the record separator
219 (default is newline "\n")
220 -s|--sort=f1[,f2] Sort data ASCII-betically on field(s)
221 --stdout Print data in original delimited format
222 (i.e., not in a table format)
223
224 --help Show brief help and quit
225 --man Show full documentation
226
227 =head1 DESCRIPTION
228
229 This program merges the fields -- not the rows -- of delimited text
230 files. That is, if several files are almost but not quite entirely
231 unlike each other in their structure (in their field names, numbers or
232 orders), this script allows you to easily unify the files into one
233 file with all the same fields. The output can be based on fields as
234 determined by the three "action" flags.
235
236 For the following examples, consider three files that contain the
237 following fields:
238
239 +------------+---------------------------------+
240 | File | Fields |
241 +------------+---------------------------------+
242 | merge1.tab | name, type, position |
243 | merge2.tab | name, type, position, lod_score |
244 | merge3.tab | name, position |
245 +------------+---------------------------------+
246
247 To list all available fields in the files and the number of times they
248 are present:
249
250 $ tabmerge --list merge*
251 +-----------+-------------------+
252 | Field | No. Times Present |
253 +-----------+-------------------+
254 | lod_score | 1 |
255 | name | 3 |
256 | position | 3 |
257 | type | 2 |
258 +-----------+-------------------+
259
260 To merge the files on the minimum overlapping fields:
261
262 $ tabmerge merge*
263 +----------+----------+
264 | name | position |
265 +----------+----------+
266 | RM104 | 2.30 |
267 | RM105 | 4.5 |
268 | TX5509 | 10.4 |
269 | UU189 | 19.0 |
270 | Xpsm122 | 3.3 |
271 | Xpsr9556 | 4.5 |
272 | DRTL | 2.30 |
273 | ALTX | 4.5 |
274 | DWRF | 10.4 |
275 +----------+----------+
276
277 To merge the files and include all the fields:
278
279 $ tabmerge --max merge*
280 +-----------+----------+----------+--------+
281 | lod_score | name | position | type |
282 +-----------+----------+----------+--------+
283 | | RM104 | 2.30 | RFLP |
284 | | RM105 | 4.5 | RFLP |
285 | | TX5509 | 10.4 | AFLP |
286 | 2.4 | UU189 | 19.0 | SSR |
287 | 1.2 | Xpsm122 | 3.3 | Marker |
288 | 1.2 | Xpsr9556 | 4.5 | Marker |
289 | | DRTL | 2.30 | |
290 | | ALTX | 4.5 | |
291 | | DWRF | 10.4 | |
292 +-----------+----------+----------+--------+
293
294 To merge and extract just the "name" and "type" fields:
295
296 $ tabmerge -f name,type merge*
297 +----------+--------+
298 | name | type |
299 +----------+--------+
300 | RM104 | RFLP |
301 | RM105 | RFLP |
302 | TX5509 | AFLP |
303 | UU189 | SSR |
304 | Xpsm122 | Marker |
305 | Xpsr9556 | Marker |
306 | DRTL | |
307 | ALTX | |
308 | DWRF | |
309 +----------+--------+
310
311 To merge the files on just the "name" and "lod_score" fields and sort on
312 the name:
313
314 $ tabmerge -f name,lod_score -s name merge*
315 +----------+-----------+
316 | name | lod_score |
317 +----------+-----------+
318 | ALTX | |
319 | DRTL | |
320 | DWRF | |
321 | RM104 | |
322 | RM105 | |
323 | TX5509 | |
324 | UU189 | 2.4 |
325 | Xpsm122 | 1.2 |
326 | Xpsr9556 | 1.2 |
327 +----------+-----------+
328
329 To do the same but mimic the original tab-delimited input:
330
331 $ tabmerge -f name,lod_score -s name --stdout merge*
332 name lod_score
333 ALTX
334 DRTL
335 DWRF
336 RM104
337 RM105
338 TX5509
339 UU189 2.4
340 Xpsm122 1.2
341 Xpsr9556 1.2
342
343 Why would you want to do this? Suppose you have several delimited text
344 files with nearly the same structure and want to create just one file
345 from them, but the fields may be in a different order in each file
346 and/or some files may contain more or fewer fields than others. (As
347 far-fetched as it may seem, it happens to the author more than he'd
348 like.)
349
350 =head1 SEE ALSO
351
352 =over 4
353
354 =item * Text::RecordParser
355
356 =item * Text::TabularDisplay
357
358 =back
359
360 =head1 AUTHOR
361
362 Ken Youens-Clark E<lt>[email protected]<gt>.
363
364 =head1 LICENSE AND COPYRIGHT
365
366 Copyright (C) 2006-10 Ken Youens-Clark. All rights reserved.
367
368 This program is free software; you can redistribute it and/or modify
369 it under the terms of the GNU General Public License as published by
370 the Free Software Foundation; version 2.
371
372 This program is distributed in the hope that it will be useful, but
373 WITHOUT ANY WARRANTY; without even the implied warranty of
374 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
375 General Public License for more details.
376
377 =cut