Debian Codegraph @

dbSwiss

raw
1 #!/usr/bin/perl -w
2 use strict;
3 use warnings;
4 use Carp qw| cluck :DEFAULT |;
5 use Getopt::Long qw(:config gnu_getopt auto_version auto_help);
6 use Pod::Usage;
7 use DBI;
8 #use File::Temp qw||;
9 use File::Path qw||;
10 use IO::File;
11 BEGIN { our $VERSION = "1.0"; }
12
13 $ENV{'PATH'} = '/usr/bin:/bin';
14 delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
15
16 our $opt;
17 my $Lok = GetOptions( $opt = { datadir => '/mnt/project/rost_db/data/swissprot', 'table' => 'dbswiss', infile => '/mnt/project/rost_db/data/swissprot/uniprot_sprot.dat' },
18 'debug!',
19 'datadir|d=s',
20 'first20!',
21 'infile|i=s',
22 'man!',
23 'quiet!',
24 'readback!',
25 'table=s',
26 'workdir|w=s'
27 );
28 if( !$Lok ){ die("Invalid arguments, please check man page.\n"); }
29
30 our $dbg = $opt->{debug};
31 our $quiet = $opt->{quiet};
32
33 if( $opt->{man} ){ pod2usage(-verbose => 2); }
34
35 if( $opt->{table} =~ /[^a-z0-9_]/io ){ confess("invalid table name '$opt->{table}'"); }
36
37 # lkajan: the below would be nice but perhaps because of the ACLs I get "Error in tempdir() using /mnt/project/rost_db/src/XXXXXXXXXX: Parent directory (/mnt/project/rost_db/src) is not writable" when it is in fact writable
38 #my $workdir = $opt->{workdir} || File::Temp::tempdir( DIR => $opt->{datadir}, CLEANUP => !$dbg );
39 my $CLEANUP = 0;
40 my $workdir = $opt->{workdir};
41 if( !$workdir ){ $CLEANUP = !$dbg; $workdir = `mktemp -d '$opt->{datadir}/XXXXXXXXXX'`; chomp( $workdir ); }
42
43 END { if( $CLEANUP ){ File::Path::rmtree( $workdir ); } }
44
45 #my $dbh = DBI->connect('DBI:DBM(RaiseError=1):type=GDBM_File;ext=;') || confess( $DBI::errstr );
46 my $dbh = DBI->connect('DBI:DBM(RaiseError=1):type=DB_File;ext=;') || confess( $DBI::errstr );
47 $dbh->{f_dir} = $workdir;
48
49 if( $dbg ){ warn( "workdir = $workdir" ); }
50
51 my $infh = IO::File->new( $opt->{infile}, 'r' ) || confess( "failed to open '$opt->{infile}': $!" );
52
53 $dbh->do( "drop table if exists $opt->{table}");
54 $dbh->do( "create table $opt->{table} ( id text, bytepos int )" ); # the value is the byte position where the record begins in the flat file
55
56 my $count = 0;
57 my $bytetop = 0;
58 my $t0 = time();
59 {
60 local $/ = "//\n";
61 while( my $rec = <$infh> )
62 {
63 #ID 002R_IIV3 Reviewed; 458 AA.
64 #ID C108B_XENLA Reviewed; 183 AA.
65 if( $rec !~ /^ID\s+(\w+)/o ){ confess("unrecognized record $count '$rec'"); }
66 my $id = lc($1);
67 $dbh->do("insert into $opt->{table} values ( ?, ? )", undef, $id, $bytetop );
68 if( $opt->{readback} )
69 {
70 my $rec = $dbh->selectrow_arrayref("select * from $opt->{table} where id = ?", undef, $id );
71 print STDERR ">>>$rec->[0] : $rec->[1]<<<\n";
72 }
73 #
74 ++$count;
75 $bytetop = tell( $infh );
76 if( $t0 != time() ){ $t0 = time(); _printcount( $count ); }
77 if( $opt->{first20} && $count > 20 ){ last; }
78 }
79 _printcount( $count ); if( !$main::quiet ){ print STDERR "\n"; }
80 }
81
82 $dbh->disconnect();
83
84 # we are done, move the db files to their final destination
85 my @resfiles = glob( "$workdir/*" );
86 system( 'mv', "-t", "$opt->{datadir}", @resfiles ) && confess("failed to mv @resfiles to $opt->{datadir}");
87
88 exit(0);
89
90 sub _printcount
91 {
92 my( $__count ) = @_;
93 if( !$main::quiet ){ print STDERR `tput hpa 0`, `tput el`, $count; }
94 }
95
96 =pod
97
98 =head1 NAME
99
100 dbSwiss - create DBM version of Swiss-Prot data
101
102 =head1 SYNOPSIS
103
104 __pkgdatadir__/dbSwiss [OPTIONS]
105
106 __pkgdatadir__/dbSwiss --datadir /data/swissprot --infile /data/swissprot/uniprot_sprot.dat
107
108 __pkgdatadir__/dbSwiss [--help] [--man]
109
110 =head1 DESCRIPTION
111
112 dbSwiss creates DBM version of Swiss-Prot data. This procedure is to replace splitSwiss.pl. splitSwiss.pl saves Swiss-Prot records in separate files resulting in over 13 million relatively tiny files that take very long to create and rsync. dbSwiss instead saves each record into a DBM database that is optimized for fast retrieval.
113
114 =head1 OPTIONS
115
116 =over
117
118 =item -d, --datadir=I<path>
119
120 directory of database files, default: '/mnt/project/rost_db/data/swissprot'
121
122 =item --debug
123
124 =item --nodebug
125
126 =item --first20
127
128 =item --nofirst20
129
130 process only first 20 records, for debugging
131
132 =item --help
133
134 =item -i, --infile=I<path>
135
136 Swiss-Prot data flatfile, default: '/mnt/project/rost_db/data/swissprot/uniprot_sprot.dat'.
137
138 =item --man
139
140 =item --quiet
141
142 =item --noquiet
143
144 do not print progress status
145
146 =item --readback
147
148 =item --noreadback
149
150 read records back after storing and print them
151
152 =item --table
153
154 name of database table and consequently the base name of database files, default: 'dbswiss'
155
156 =item --version
157
158 =item -w, --workdir=I<path>
159
160 Optional working directory. Automatically created and removed if not defined.
161
162 =back
163
164 =head1 AUTHOR
165
166 Laszlo Kajan <[email protected]>
167
168 =cut
169
170 # vim:et:ts=2:ai:
171