| 1 |
#!/usr/bin/perl -w |
| 2 |
use strict; |
| 3 |
use warnings; |
| 4 |
use Carp qw| cluck :DEFAULT |; |
| 5 |
use Getopt::Long qw(:config gnu_getopt auto_version auto_help); |
| 6 |
use Pod::Usage; |
| 7 |
use DBI; |
| 8 |
#use File::Temp qw||; |
| 9 |
use File::Path qw||; |
| 10 |
use IO::File; |
| 11 |
BEGIN { our $VERSION = "1.0"; } |
| 12 |
|
| 13 |
$ENV{'PATH'} = '/usr/bin:/bin'; |
| 14 |
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'}; |
| 15 |
|
| 16 |
our $opt; |
| 17 |
my $Lok = GetOptions( $opt = { datadir => '/mnt/project/rost_db/data/swissprot', 'table' => 'dbswiss', infile => '/mnt/project/rost_db/data/swissprot/uniprot_sprot.dat' }, |
| 18 |
'debug!', |
| 19 |
'datadir|d=s', |
| 20 |
'first20!', |
| 21 |
'infile|i=s', |
| 22 |
'man!', |
| 23 |
'quiet!', |
| 24 |
'readback!', |
| 25 |
'table=s', |
| 26 |
'workdir|w=s' |
| 27 |
); |
| 28 |
if( !$Lok ){ die("Invalid arguments, please check man page.\n"); } |
| 29 |
|
| 30 |
our $dbg = $opt->{debug}; |
| 31 |
our $quiet = $opt->{quiet}; |
| 32 |
|
| 33 |
if( $opt->{man} ){ pod2usage(-verbose => 2); } |
| 34 |
|
| 35 |
if( $opt->{table} =~ /[^a-z0-9_]/io ){ confess("invalid table name '$opt->{table}'"); } |
| 36 |
|
| 37 |
# lkajan: the below would be nice but perhaps because of the ACLs I get "Error in tempdir() using /mnt/project/rost_db/src/XXXXXXXXXX: Parent directory (/mnt/project/rost_db/src) is not writable" when it is in fact writable |
| 38 |
#my $workdir = $opt->{workdir} || File::Temp::tempdir( DIR => $opt->{datadir}, CLEANUP => !$dbg ); |
| 39 |
my $CLEANUP = 0; |
| 40 |
my $workdir = $opt->{workdir}; |
| 41 |
if( !$workdir ){ $CLEANUP = !$dbg; $workdir = `mktemp -d '$opt->{datadir}/XXXXXXXXXX'`; chomp( $workdir ); } |
| 42 |
|
| 43 |
END { if( $CLEANUP ){ File::Path::rmtree( $workdir ); } } |
| 44 |
|
| 45 |
#my $dbh = DBI->connect('DBI:DBM(RaiseError=1):type=GDBM_File;ext=;') || confess( $DBI::errstr ); |
| 46 |
my $dbh = DBI->connect('DBI:DBM(RaiseError=1):type=DB_File;ext=;') || confess( $DBI::errstr ); |
| 47 |
$dbh->{f_dir} = $workdir; |
| 48 |
|
| 49 |
if( $dbg ){ warn( "workdir = $workdir" ); } |
| 50 |
|
| 51 |
my $infh = IO::File->new( $opt->{infile}, 'r' ) || confess( "failed to open '$opt->{infile}': $!" ); |
| 52 |
|
| 53 |
$dbh->do( "drop table if exists $opt->{table}"); |
| 54 |
$dbh->do( "create table $opt->{table} ( id text, bytepos int )" ); # the value is the byte position where the record begins in the flat file |
| 55 |
|
| 56 |
my $count = 0; |
| 57 |
my $bytetop = 0; |
| 58 |
my $t0 = time(); |
| 59 |
{ |
| 60 |
local $/ = "//\n"; |
| 61 |
while( my $rec = <$infh> ) |
| 62 |
{ |
| 63 |
#ID 002R_IIV3 Reviewed; 458 AA. |
| 64 |
#ID C108B_XENLA Reviewed; 183 AA. |
| 65 |
if( $rec !~ /^ID\s+(\w+)/o ){ confess("unrecognized record $count '$rec'"); } |
| 66 |
my $id = lc($1); |
| 67 |
$dbh->do("insert into $opt->{table} values ( ?, ? )", undef, $id, $bytetop ); |
| 68 |
if( $opt->{readback} ) |
| 69 |
{ |
| 70 |
my $rec = $dbh->selectrow_arrayref("select * from $opt->{table} where id = ?", undef, $id ); |
| 71 |
print STDERR ">>>$rec->[0] : $rec->[1]<<<\n"; |
| 72 |
} |
| 73 |
# |
| 74 |
++$count; |
| 75 |
$bytetop = tell( $infh ); |
| 76 |
if( $t0 != time() ){ $t0 = time(); _printcount( $count ); } |
| 77 |
if( $opt->{first20} && $count > 20 ){ last; } |
| 78 |
} |
| 79 |
_printcount( $count ); if( !$main::quiet ){ print STDERR "\n"; } |
| 80 |
} |
| 81 |
|
| 82 |
$dbh->disconnect(); |
| 83 |
|
| 84 |
# we are done, move the db files to their final destination |
| 85 |
my @resfiles = glob( "$workdir/*" ); |
| 86 |
system( 'mv', "-t", "$opt->{datadir}", @resfiles ) && confess("failed to mv @resfiles to $opt->{datadir}"); |
| 87 |
|
| 88 |
exit(0); |
| 89 |
|
| 90 |
sub _printcount |
| 91 |
{ |
| 92 |
my( $__count ) = @_; |
| 93 |
if( !$main::quiet ){ print STDERR `tput hpa 0`, `tput el`, $count; } |
| 94 |
} |
| 95 |
|
| 96 |
=pod |
| 97 |
|
| 98 |
=head1 NAME |
| 99 |
|
| 100 |
dbSwiss - create DBM version of Swiss-Prot data |
| 101 |
|
| 102 |
=head1 SYNOPSIS |
| 103 |
|
| 104 |
__pkgdatadir__/dbSwiss [OPTIONS] |
| 105 |
|
| 106 |
__pkgdatadir__/dbSwiss --datadir /data/swissprot --infile /data/swissprot/uniprot_sprot.dat |
| 107 |
|
| 108 |
__pkgdatadir__/dbSwiss [--help] [--man] |
| 109 |
|
| 110 |
=head1 DESCRIPTION |
| 111 |
|
| 112 |
dbSwiss creates DBM version of Swiss-Prot data. This procedure is to replace splitSwiss.pl. splitSwiss.pl saves Swiss-Prot records in separate files resulting in over 13 million relatively tiny files that take very long to create and rsync. dbSwiss instead saves each record into a DBM database that is optimized for fast retrieval. |
| 113 |
|
| 114 |
=head1 OPTIONS |
| 115 |
|
| 116 |
=over |
| 117 |
|
| 118 |
=item -d, --datadir=I<path> |
| 119 |
|
| 120 |
directory of database files, default: '/mnt/project/rost_db/data/swissprot' |
| 121 |
|
| 122 |
=item --debug |
| 123 |
|
| 124 |
=item --nodebug |
| 125 |
|
| 126 |
=item --first20 |
| 127 |
|
| 128 |
=item --nofirst20 |
| 129 |
|
| 130 |
process only first 20 records, for debugging |
| 131 |
|
| 132 |
=item --help |
| 133 |
|
| 134 |
=item -i, --infile=I<path> |
| 135 |
|
| 136 |
Swiss-Prot data flatfile, default: '/mnt/project/rost_db/data/swissprot/uniprot_sprot.dat'. |
| 137 |
|
| 138 |
=item --man |
| 139 |
|
| 140 |
=item --quiet |
| 141 |
|
| 142 |
=item --noquiet |
| 143 |
|
| 144 |
do not print progress status |
| 145 |
|
| 146 |
=item --readback |
| 147 |
|
| 148 |
=item --noreadback |
| 149 |
|
| 150 |
read records back after storing and print them |
| 151 |
|
| 152 |
=item --table |
| 153 |
|
| 154 |
name of database table and consequently the base name of database files, default: 'dbswiss' |
| 155 |
|
| 156 |
=item --version |
| 157 |
|
| 158 |
=item -w, --workdir=I<path> |
| 159 |
|
| 160 |
Optional working directory. Automatically created and removed if not defined. |
| 161 |
|
| 162 |
=back |
| 163 |
|
| 164 |
=head1 AUTHOR |
| 165 |
|
| 166 |
Laszlo Kajan <[email protected]> |
| 167 |
|
| 168 |
=cut |
| 169 |
|
| 170 |
# vim:et:ts=2:ai: |
| 171 |
|