| 1 |
#!perl |
| 2 |
|
| 3 |
use strict; |
| 4 |
use warnings; |
| 5 |
use version; |
| 6 |
use English qw( -no_match_vars ); |
| 7 |
use File::Basename; |
| 8 |
use Getopt::Long; |
| 9 |
use Pod::Usage; |
| 10 |
use Readonly; |
| 11 |
use Text::RecordParser; |
| 12 |
use Text::TabularDisplay; |
| 13 |
|
| 14 |
Readonly our $VERSION => 1.12; |
| 15 |
Readonly my $COMMA => q{,}; |
| 16 |
Readonly my $COMMA_SPACE => q{, }; |
| 17 |
Readonly my $DASH => q{-}; |
| 18 |
Readonly my $EMPTY_STR => q{}; |
| 19 |
Readonly my $TAB => qq{\t}; |
| 20 |
Readonly my $NEWLINE => qq{\n}; |
| 21 |
|
| 22 |
my $fs = $TAB; |
| 23 |
my $rs = $NEWLINE; |
| 24 |
my ( $fields, $min, $max, $list, $sort, $stdout, |
| 25 |
$help, $man_page, $show_version ); |
| 26 |
|
| 27 |
GetOptions( |
| 28 |
'f|fields:s' => \$fields, |
| 29 |
'help' => \$help, |
| 30 |
'fs:s' => \$fs, |
| 31 |
'l|list' => \$list, |
| 32 |
'man' => \$man_page, |
| 33 |
'min' => \$min, |
| 34 |
'max' => \$max, |
| 35 |
'rs:s' => \$rs, |
| 36 |
's|sort:s' => \$sort, |
| 37 |
'stdout' => \$stdout, |
| 38 |
'version' => \$show_version, |
| 39 |
) or pod2usage; |
| 40 |
|
| 41 |
if ( $help || $man_page ) { |
| 42 |
pod2usage({ |
| 43 |
-exitval => 0, |
| 44 |
-verbose => $man_page ? 2 : 1 |
| 45 |
}); |
| 46 |
}; |
| 47 |
|
| 48 |
if ( $show_version ) { |
| 49 |
my $prog = basename( $PROGRAM_NAME ); |
| 50 |
print "$prog $VERSION\n"; |
| 51 |
exit 0; |
| 52 |
} |
| 53 |
|
| 54 |
if ( scalar @ARGV < 2 ) { |
| 55 |
pod2usage('Please supply two or more files'); |
| 56 |
} |
| 57 |
|
| 58 |
my @files = @ARGV or pod2usage('No input files'); |
| 59 |
|
| 60 |
$min = 1 unless $max or $fields; |
| 61 |
|
| 62 |
my $p = Text::RecordParser->new( |
| 63 |
field_separator => $fs, |
| 64 |
record_separator => $rs, |
| 65 |
); |
| 66 |
|
| 67 |
if ( $list ) { |
| 68 |
list( $p, @files ); |
| 69 |
} |
| 70 |
elsif ( $min || $max || $fields ) { |
| 71 |
intersection( |
| 72 |
fields => $fields, |
| 73 |
files => \@files, |
| 74 |
parser => $p, |
| 75 |
sort => $sort, |
| 76 |
type => $min ? 'min' : 'max', |
| 77 |
); |
| 78 |
} |
| 79 |
else { |
| 80 |
pod2usage(q[Can't figure out what to do.]); |
| 81 |
} |
| 82 |
|
| 83 |
# ------------------------------------------------------------------- |
| 84 |
sub intersection { |
| 85 |
my %args = @_; |
| 86 |
my $p = $args{'parser'}; |
| 87 |
my $type = $args{'type'}; |
| 88 |
my $files = $args{'files'}; |
| 89 |
my $fields = $args{'fields'} || $EMPTY_STR; # optional |
| 90 |
my $sort = $args{'sort'} || $EMPTY_STR; # optional |
| 91 |
|
| 92 |
my %fields; |
| 93 |
for my $file ( @$files ) { |
| 94 |
$p->filename( $file ); |
| 95 |
$p->bind_header; |
| 96 |
$fields{ $_ }++ for $p->field_list; |
| 97 |
} |
| 98 |
|
| 99 |
my @intersect; |
| 100 |
if ( $fields ) { |
| 101 |
@intersect = map { s/^\s+|\s+$//g; $_ } split /$COMMA/, $fields; |
| 102 |
die_if_bad( \%fields, \@intersect ); |
| 103 |
} |
| 104 |
else { |
| 105 |
my $no_files = scalar @$files; |
| 106 |
for my $fld ( keys %fields ) { |
| 107 |
push @intersect, $fld if |
| 108 |
( $type eq 'max' ) || |
| 109 |
( $type eq 'min' && $fields{ $fld } == $no_files ) |
| 110 |
; |
| 111 |
} |
| 112 |
|
| 113 |
@intersect = sort @intersect; |
| 114 |
} |
| 115 |
|
| 116 |
die "No intersection!\n" unless @intersect; |
| 117 |
|
| 118 |
my @data; |
| 119 |
for my $file ( @$files ) { |
| 120 |
$p->filename( $file ); |
| 121 |
$p->bind_header; |
| 122 |
while ( my $rec = $p->fetchrow_hashref ) { |
| 123 |
push @data, { map { $_, $rec->{ $_ } } @intersect }; |
| 124 |
} |
| 125 |
} |
| 126 |
|
| 127 |
if ( my @sort = map { s/^\s+|\s+$//g; $_ } split /$COMMA/, $sort ) { |
| 128 |
die_if_bad( \%fields, \@sort ); |
| 129 |
|
| 130 |
@data = |
| 131 |
map { $_->[1] } |
| 132 |
sort { $a->[0] cmp $b->[0] } |
| 133 |
map { [ join($DASH, @{ $_ }{ @sort } ), $_ ] } |
| 134 |
@data; |
| 135 |
} |
| 136 |
|
| 137 |
if ( $stdout ) { |
| 138 |
print join( $fs, @intersect ), $rs; |
| 139 |
for my $rec ( @data ) { |
| 140 |
print join( $fs, |
| 141 |
map { defined $rec->{$_} ? $rec->{$_} : $EMPTY_STR } @intersect |
| 142 |
), $rs; |
| 143 |
} |
| 144 |
} |
| 145 |
else { |
| 146 |
my $tab = Text::TabularDisplay->new( @intersect ); |
| 147 |
for my $rec ( @data ) { |
| 148 |
$tab->add( map { $rec->{ $_ } } @intersect ); |
| 149 |
} |
| 150 |
print $tab->render, $NEWLINE; |
| 151 |
} |
| 152 |
} |
| 153 |
|
| 154 |
# ------------------------------------------------------------------- |
| 155 |
sub list { |
| 156 |
my ( $p, @files ) = @_; |
| 157 |
|
| 158 |
my %fields; |
| 159 |
for my $file ( @files ) { |
| 160 |
$p->filename( $file ); |
| 161 |
$p->bind_header; |
| 162 |
$fields{ $_ }++ for $p->field_list; |
| 163 |
} |
| 164 |
|
| 165 |
my $tab = Text::TabularDisplay->new('Field', 'No. Times Present'); |
| 166 |
$tab->add( $_, $fields{ $_ } ) for sort keys %fields; |
| 167 |
print $tab->render, $NEWLINE; |
| 168 |
} |
| 169 |
|
| 170 |
# ------------------------------------------------------------------- |
| 171 |
sub die_if_bad { |
| 172 |
my ( $fields, $check ) = @_; |
| 173 |
|
| 174 |
my @bad; |
| 175 |
for my $fld ( @$check ) { |
| 176 |
push @bad, $fld unless $fields->{ $fld }; |
| 177 |
} |
| 178 |
|
| 179 |
if ( @bad ) { |
| 180 |
die sprintf "Bad field name%s: %s$NEWLINE", |
| 181 |
scalar @bad > 1 ? 's' : $EMPTY_STR, |
| 182 |
join $COMMA_SPACE, @bad |
| 183 |
; |
| 184 |
} |
| 185 |
else { |
| 186 |
return 1; |
| 187 |
} |
| 188 |
} |
| 189 |
|
| 190 |
__END__ |
| 191 |
|
| 192 |
# ------------------------------------------------------------------- |
| 193 |
|
| 194 |
# $Id: tabmerge,v 1.11 2006/03/07 17:20:00 kclark Exp $ |
| 195 |
|
| 196 |
=pod |
| 197 |
|
| 198 |
=head1 NAME |
| 199 |
|
| 200 |
tabmerge - unify delimited files on common fields |
| 201 |
|
| 202 |
=head1 SYNOPSIS |
| 203 |
|
| 204 |
tabmerge [action] [options] file1 file2 [...] |
| 205 |
|
| 206 |
Actions: |
| 207 |
|
| 208 |
--min Take only fields present in all files [DEFAULT] |
| 209 |
--max Take all fields present |
| 210 |
-f|--fields=f1[,f2] Take only the fields mentioned in the |
| 211 |
comma-separated list |
| 212 |
|
| 213 |
Options: |
| 214 |
|
| 215 |
-l|--list List available fields |
| 216 |
--fs=x Use "x" as the field separator |
| 217 |
(default is tab "\t") |
| 218 |
--rs=x Use "x" as the record separator |
| 219 |
(default is newline "\n") |
| 220 |
-s|--sort=f1[,f2] Sort data ASCII-betically on field(s) |
| 221 |
--stdout Print data in original delimited format |
| 222 |
(i.e., not in a table format) |
| 223 |
|
| 224 |
--help Show brief help and quit |
| 225 |
--man Show full documentation |
| 226 |
|
| 227 |
=head1 DESCRIPTION |
| 228 |
|
| 229 |
This program merges the fields -- not the rows -- of delimited text |
| 230 |
files. That is, if several files are almost but not quite entirely |
| 231 |
unlike each other in their structure (in their field names, numbers or |
| 232 |
orders), this script allows you to easily unify the files into one |
| 233 |
file with all the same fields. The output can be based on fields as |
| 234 |
determined by the three "action" flags. |
| 235 |
|
| 236 |
For the following examples, consider three files that contain the |
| 237 |
following fields: |
| 238 |
|
| 239 |
+------------+---------------------------------+ |
| 240 |
| File | Fields | |
| 241 |
+------------+---------------------------------+ |
| 242 |
| merge1.tab | name, type, position | |
| 243 |
| merge2.tab | name, type, position, lod_score | |
| 244 |
| merge3.tab | name, position | |
| 245 |
+------------+---------------------------------+ |
| 246 |
|
| 247 |
To list all available fields in the files and the number of times they |
| 248 |
are present: |
| 249 |
|
| 250 |
$ tabmerge --list merge* |
| 251 |
+-----------+-------------------+ |
| 252 |
| Field | No. Times Present | |
| 253 |
+-----------+-------------------+ |
| 254 |
| lod_score | 1 | |
| 255 |
| name | 3 | |
| 256 |
| position | 3 | |
| 257 |
| type | 2 | |
| 258 |
+-----------+-------------------+ |
| 259 |
|
| 260 |
To merge the files on the minimum overlapping fields: |
| 261 |
|
| 262 |
$ tabmerge merge* |
| 263 |
+----------+----------+ |
| 264 |
| name | position | |
| 265 |
+----------+----------+ |
| 266 |
| RM104 | 2.30 | |
| 267 |
| RM105 | 4.5 | |
| 268 |
| TX5509 | 10.4 | |
| 269 |
| UU189 | 19.0 | |
| 270 |
| Xpsm122 | 3.3 | |
| 271 |
| Xpsr9556 | 4.5 | |
| 272 |
| DRTL | 2.30 | |
| 273 |
| ALTX | 4.5 | |
| 274 |
| DWRF | 10.4 | |
| 275 |
+----------+----------+ |
| 276 |
|
| 277 |
To merge the files and include all the fields: |
| 278 |
|
| 279 |
$ tabmerge --max merge* |
| 280 |
+-----------+----------+----------+--------+ |
| 281 |
| lod_score | name | position | type | |
| 282 |
+-----------+----------+----------+--------+ |
| 283 |
| | RM104 | 2.30 | RFLP | |
| 284 |
| | RM105 | 4.5 | RFLP | |
| 285 |
| | TX5509 | 10.4 | AFLP | |
| 286 |
| 2.4 | UU189 | 19.0 | SSR | |
| 287 |
| 1.2 | Xpsm122 | 3.3 | Marker | |
| 288 |
| 1.2 | Xpsr9556 | 4.5 | Marker | |
| 289 |
| | DRTL | 2.30 | | |
| 290 |
| | ALTX | 4.5 | | |
| 291 |
| | DWRF | 10.4 | | |
| 292 |
+-----------+----------+----------+--------+ |
| 293 |
|
| 294 |
To merge and extract just the "name" and "type" fields: |
| 295 |
|
| 296 |
$ tabmerge -f name,type merge* |
| 297 |
+----------+--------+ |
| 298 |
| name | type | |
| 299 |
+----------+--------+ |
| 300 |
| RM104 | RFLP | |
| 301 |
| RM105 | RFLP | |
| 302 |
| TX5509 | AFLP | |
| 303 |
| UU189 | SSR | |
| 304 |
| Xpsm122 | Marker | |
| 305 |
| Xpsr9556 | Marker | |
| 306 |
| DRTL | | |
| 307 |
| ALTX | | |
| 308 |
| DWRF | | |
| 309 |
+----------+--------+ |
| 310 |
|
| 311 |
To merge the files on just the "name" and "lod_score" fields and sort on |
| 312 |
the name: |
| 313 |
|
| 314 |
$ tabmerge -f name,lod_score -s name merge* |
| 315 |
+----------+-----------+ |
| 316 |
| name | lod_score | |
| 317 |
+----------+-----------+ |
| 318 |
| ALTX | | |
| 319 |
| DRTL | | |
| 320 |
| DWRF | | |
| 321 |
| RM104 | | |
| 322 |
| RM105 | | |
| 323 |
| TX5509 | | |
| 324 |
| UU189 | 2.4 | |
| 325 |
| Xpsm122 | 1.2 | |
| 326 |
| Xpsr9556 | 1.2 | |
| 327 |
+----------+-----------+ |
| 328 |
|
| 329 |
To do the same but mimic the original tab-delimited input: |
| 330 |
|
| 331 |
$ tabmerge -f name,lod_score -s name --stdout merge* |
| 332 |
name lod_score |
| 333 |
ALTX |
| 334 |
DRTL |
| 335 |
DWRF |
| 336 |
RM104 |
| 337 |
RM105 |
| 338 |
TX5509 |
| 339 |
UU189 2.4 |
| 340 |
Xpsm122 1.2 |
| 341 |
Xpsr9556 1.2 |
| 342 |
|
| 343 |
Why would you want to do this? Suppose you have several delimited text |
| 344 |
files with nearly the same structure and want to create just one file |
| 345 |
from them, but the fields may be in a different order in each file |
| 346 |
and/or some files may contain more or fewer fields than others. (As |
| 347 |
far-fetched as it may seem, it happens to the author more than he'd |
| 348 |
like.) |
| 349 |
|
| 350 |
=head1 SEE ALSO |
| 351 |
|
| 352 |
=over 4 |
| 353 |
|
| 354 |
=item * Text::RecordParser |
| 355 |
|
| 356 |
=item * Text::TabularDisplay |
| 357 |
|
| 358 |
=back |
| 359 |
|
| 360 |
=head1 AUTHOR |
| 361 |
|
| 362 |
Ken Youens-Clark E<lt>[email protected]<gt>. |
| 363 |
|
| 364 |
=head1 LICENSE AND COPYRIGHT |
| 365 |
|
| 366 |
Copyright (C) 2006-10 Ken Youens-Clark. All rights reserved. |
| 367 |
|
| 368 |
This program is free software; you can redistribute it and/or modify |
| 369 |
it under the terms of the GNU General Public License as published by |
| 370 |
the Free Software Foundation; version 2. |
| 371 |
|
| 372 |
This program is distributed in the hope that it will be useful, but |
| 373 |
WITHOUT ANY WARRANTY; without even the implied warranty of |
| 374 |
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 375 |
General Public License for more details. |
| 376 |
|
| 377 |
=cut |