Debian Codegraph @

scripts/asub

raw
1 #!/usr/bin/perl -w
2
3 # Author: lh3
4
5 use strict;
6 use warnings;
7 use Getopt::Std;
8 use Cwd qw/abs_path/;
9 use Sys::Hostname qw/hostname/;
10
11 my $version = "0.1.5";
12
13 my %opts = (R=>'', j=>'', k=>0, q=>'', c=>'', w=>'', M=>'', n=>'');
14 getopts('R:j:k:q:c:pw:M:n:', \%opts);
15 &usage if (-t STDIN && @ARGV == 0);
16 my $is_run = ($opts{k} > 0 && $opts{j})? 1 : 0;
17 if (!$is_run) { # prepare to run
18 my $is_sge = &which('bsub')? 0 : 1;
19 $opts{j} ||= sprintf("asub_$$%.3d", rand(1000));
20 my ($fh, $K);
21 $K = 0;
22 open($fh, ">$opts{j}.sh") || die;
23 while (<>) {
24 s/&\s*$//; # remove "&" at the end of the command
25 next if (/^\s*$/);
26 print $fh $_;
27 ++$K;
28 }
29 close($fh);
30 $_ = $opts{j};
31 my $cmd;
32 $opts{c} = $K if ($opts{c} && $opts{c} > $K);
33 $opts{q} = qq/-q $opts{q}/ if ($opts{q});
34 if (!$is_sge) { # LSF
35 $opts{R} = qq/-R"$opts{R}"/ if ($opts{R});
36 $opts{c} = qq/\%$opts{c}/ if ($opts{c});
37 $opts{w} = qq/-w "$opts{w}"/ if ($opts{w});
38 $opts{n} = qq/-n $opts{n}/ if ($opts{n});
39 if ($opts{M}) {
40 if ($opts{R}) {
41 chop($opts{R}); # chop the tailing "
42 $opts{R} .= qq/ select[mem>$opts{M}] rusage[mem=$opts{M}]"/;
43 } else {
44 $opts{R} = qq/-R"select[mem>$opts{M}] rusage[mem=$opts{M}]"/;
45 }
46 $opts{R} .= ' -M ' . ($opts{M}*1000);
47 }
48 $cmd = qq(mkdir -p $_.out $_.err && echo '$0 -j $_.sh -k \${LSB_JOBINDEX}' | bsub -J$_)
49 . qq("[1-$K]$opts{c}" -o $_.out/\%I.out -e $_.err/\%I.err $opts{R} $opts{q} $opts{w});
50 } else { # SGE
51 $opts{R} = qq/-l "$opts{R}"/ if ($opts{R});
52 $opts{w} = qq/-hold_jid $opts{w}/ if ($opts{w});
53 my $asub = abs_path($0);
54 $cmd = qq(mkdir -p $_.out $_.err && echo '$asub -j $_.sh -k \${SGE_TASK_ID}' | qsub -N $_ -cwd )
55 . qq(-t 1-$K -o '$_.out/\$TASK_ID.out' -e '$_.err/\$TASK_ID.err' $opts{R} $opts{q} $opts{w} $opts{n});
56 }
57 defined($opts{p})? (print "$cmd\n") : system($cmd);
58 } else { # run the command
59 warn("[asub] Hostname: ".hostname.", Arch-OS: ".&get_cpu_sys."\n");
60 my $fh;
61 my $k = 0;
62 open($fh, $opts{j}) || die;
63 while (<$fh>) {
64 chomp;
65 last if ((++$k) == $opts{k});
66 }
67 my $cmd = $_;
68 close($fh);
69 warn("[asub] Command: $_\n");
70 warn("[asub] --- BEGIN OF COMMAND STDERR ---\n");
71 my $ret = system($_);
72 warn("[asub] --- END OF COMMAND STDERR ---\n");
73 warn("[asub] Return value: $ret\n");
74 exit($ret);
75 }
76
77 sub get_cpu_sys {
78 my $dir = `uname -p 2>/dev/null`;
79 $dir = `uname -m 2>/dev/null` if (!$dir || $dir =~ "unknown");
80 $dir .= '-'.`uname -s`;
81 $dir = lc($dir);
82 $dir =~ s/\s//g;
83 return $dir;
84 }
85
86 sub which {
87 my $file = shift;
88 my $path = (@_)? shift : $ENV{PATH};
89 return if (!defined($path));
90 foreach my $x (split(":", $path)) {
91 $x =~ s/\/$//;
92 return "$x/$file" if (-x "$x/$file");
93 }
94 return;
95 }
96
97 sub usage {
98 die qq(
99 Program: asub (Array Job submission with bsub/qsub)
100 Version: $version
101 Contact: Heng Li <lh3\@sanger.ac.uk>\n
102 Usage: asub [options] <cmd_file>\n
103 Options: -R STR resources string [null]
104 -w STR job dependency [null]
105 -q STR queue name [default queue]
106 -j STR job name [auto]
107 -c INT number of processors (LSF only) [max]
108 -M INT maximum memory in MB (Sanger LSF only) [null]
109 -n INT # CPUs for a parallel job (LSF only) [null]
110 -p print the submission command only
111
112 Note: For option -R and -w, SGE and LSF have different syntax. Please
113 check SGE/LSF manual page for details. Here are some examples for
114 LSF:
115
116 -R "select[type==X86_64&&mem>800] rusage[mem=800]"
117 -w "done(my_job_001)"
118
119 And for SGE:
120
121 -R h_cpu=86400,h_data=1000000000
122 -w my_job_001
123
124 );
125 }