Debian Codegraph @

bin/schwartzmon

raw
1 #!/usr/bin/perl
2 use strict;
3 use DBI;
4 use Getopt::Long;
5
6 my $dbname = "schwartz";
7 my $user = "root";
8 my $pass = "";
9 my $job = "";
10 my $max_age = 0;
11 my $max_count = 0;
12
13 =head1 NAME
14
15 schwartzmon - monitor The Schwartz
16
17 =head1 USAGE
18
19 Type
20
21 schwartzmon --help
22
23 to get full usage.
24
25 =cut
26
27 sub usage {
28 die <<USAGE;
29 Usage: schwartzmon <command> [OPTS]
30
31 Possible commands:
32 queues View past-due job queue depths. (default cmd)
33 errors View errors.
34
35 Global options:
36 --job=<JOBNAME> Only look at one specific job name. Else all are considered.
37 --user=<user> Connect to the database as this user
38 --pass=<pass> Connect to the database with this password
39 --database=<db> Connect to this database
40 --dsn=<dsn> Connect to the database using this DSN
41
42 Options for 'queues' command:
43 --maxage=<n> Don't complain if age of overdue job queue is <= 'n'
44 --maxcount=<n> Don't complain if depth of overdue job queue is <= 'n'
45
46 Options for 'errors' command:
47 --follow | -f Like 'tail -f' for tracking the error log
48 --last=n Show last 'n' errors from log
49 --inlast=n Show errors in last 'n' seconds
50
51 Verbosity:
52 if no alerts, nothing is printed, and exit status is 0.
53
54 Exit status:
55 0 if no alerts,
56 non-zero if there are alerts, in which case the alerts are printed.
57
58 USAGE
59
60 }
61
62 my $opt_help = 0;
63 my ( $opt_follow, $opt_last, $opt_inlast, $opt_func, $dsn );
64 usage()
65 unless GetOptions(
66 "job=s" => \$job,
67 "maxage=i" => \$max_age,
68 "maxcount=i" => \$max_count,
69 "help" => \$opt_help,
70 "follow|f" => \$opt_follow,
71 "last=i" => \$opt_last,
72 "inlast=i" => \$opt_inlast,
73 "user=s" => \$user,
74 "pass=s" => \$pass,
75 "dsn=s" => \$dsn,
76 "database=s" => \$dbname,
77 "func=s" => \$opt_func,
78 );
79 usage() if $opt_help;
80
81 my $cmd = shift || "queues";
82 usage() unless $cmd =~ /^queues|errors$/;
83
84 my $dbset = DBSet->new;
85
86 $dsn ||= "DBI:mysql:$dbname";
87
88 $dbset->add( DBHandle->new( { dsn => $dsn, user => $user, pass => $pass } ) );
89
90 if ( $cmd eq "queues" ) { queues($dbset); }
91 if ( $cmd eq "errors" ) { errors($dbset); }
92 exit 0;
93
94 #################
95
96 sub queues {
97 my $dbs = shift;
98 my $some_alert = 0;
99 $dbs->foreach(
100 sub {
101 my $db = shift;
102 my $dbh = $db->dbh or next;
103
104 my $funcmap = $dbh->selectall_hashref(
105 "SELECT funcid, funcname FROM funcmap", "funcid" );
106
107 foreach my $funcid (
108 sort { $funcmap->{$a}{funcname} cmp $funcmap->{$b}{funcname} }
109 keys %$funcmap
110 )
111 {
112 my $funcname = $funcmap->{$funcid}{funcname};
113 next if $job && $funcname ne $job;
114
115 my $now = time();
116 my $inf = $dbh->selectrow_hashref(
117 "SELECT COUNT(*) as 'ct', MIN(run_after) 'oldest' FROM job WHERE funcid=? AND run_after <= $now",
118 undef, $funcid
119 );
120 my $behind = $inf->{ct} ? ( $now - $inf->{oldest} ) : 0;
121
122 # okay by default, then we apply rules:
123 my $okay = 1;
124 $okay = 0 if $behind > $max_age;
125 $okay = 0 if $inf->{ct} > $max_count;
126 next if $okay;
127 $some_alert = 1;
128
129 print "$funcname\n";
130 print " outstanding: $inf->{ct}\n";
131 print " behind_secs: $behind\n";
132 }
133 }
134 );
135 exit( $some_alert ? 1 : 0 );
136 }
137
138 sub errors {
139 my $dbs = shift;
140
141 if ($opt_follow) {
142 follow_errors($dbs);
143 }
144
145 $opt_last = 100 unless $opt_last || $opt_inlast;
146
147 my @rows;
148 $dbs->foreach(
149 sub {
150 my $db = shift;
151 my $dbh = $db->dbh
152 or next;
153
154 my $extra_where = '';
155 if ($opt_func) {
156 my $funcid = $db->funcid_of_func($opt_func) || 0;
157 $extra_where = "AND funcid=$funcid";
158 }
159
160 my $sql;
161 if ($opt_last) {
162 $sql
163 = "SELECT error_time, jobid, message FROM error WHERE 1=1 $extra_where "
164 . "ORDER BY error_time DESC LIMIT $opt_last";
165 }
166 elsif ($opt_inlast) {
167 my $since = time() - $opt_inlast;
168 $sql
169 = "SELECT error_time, jobid, message FROM error WHERE error_time >= $since $extra_where "
170 . "ORDER BY error_time LIMIT 50000";
171 }
172
173 my $sth = $dbh->prepare($sql);
174 $sth->execute;
175 push @rows, $_ while $_ = $sth->fetchrow_hashref;
176 }
177 );
178
179 @rows = sort { $a->{error_time} <=> $b->{error_time} } @rows;
180 if ( $opt_last && @rows > $opt_last ) {
181 shift @rows while @rows > $opt_last;
182 }
183
184 foreach my $r (@rows) {
185 print_error($r);
186 }
187
188 }
189
190 sub follow_errors {
191 my $dbs = shift;
192
193 while (1) {
194 $dbs->foreach(
195 sub {
196 my $db = shift;
197 my $dbh = $db->dbh
198 or next;
199 my $notes = $db->notes;
200
201 my $lastmax = $notes->{lastmax} || time();
202 my $seen = $notes->{seen} ||= {};
203
204 my $extra_where = '';
205 if ($opt_func) {
206 my $funcid = $db->funcid_of_func($opt_func) || 0;
207 $extra_where = "AND funcid=$funcid";
208 }
209
210 my $sth
211 = $dbh->prepare(
212 "SELECT error_time, jobid, message FROM error WHERE error_time >= ? $extra_where ORDER BY error_time"
213 );
214 $sth->execute($lastmax);
215 my @errors;
216 push @errors, $_ while $_ = $sth->fetchrow_hashref;
217
218 my $newmax = $lastmax;
219 foreach my $r (@errors) {
220 my $sig = join( ",", map { $_, $r->{$_} } sort keys %$r );
221 next if $seen->{$sig};
222 $seen->{$sig} = $r->{error_time};
223 print_error($r);
224 $newmax = $r->{error_time} if $r->{error_time} > $newmax;
225 }
226
227 $notes->{lastmax} = $newmax;
228
229 foreach my $sig ( keys %$seen ) {
230 my $time = $seen->{$sig};
231 delete $seen->{$sig} if $time < $newmax;
232 }
233
234 }
235 );
236 sleep 1;
237 }
238 }
239
240 sub print_error {
241 my $r = shift;
242 my $msg = $r->{message};
243 $msg =~ s/\s+$//g;
244 printf scalar( localtime( $r->{error_time} ) ) . " [$r->{jobid}]: $msg\n";
245 }
246
247 package DBSet;
248
249 sub new {
250 my ( $this, $args ) = @_;
251 my $class = ref($this) || $this;
252 return bless {}, $class;
253 }
254
255 sub add {
256 my ( $self, $db ) = @_;
257 push @$self, $db;
258 }
259
260 sub foreach {
261 my ( $self, $cb ) = @_;
262 foreach my $dbh (@$self) {
263 $cb->($dbh);
264 }
265 }
266
267 package DBHandle;
268
269 sub new {
270 my ( $class, $dbinf ) = @_;
271 return bless $dbinf, $class;
272 }
273
274 sub notes {
275 my $self = shift;
276 return $self->{notes} ||= {};
277 }
278
279 # returns DBI handle
280 sub dbh {
281 my $self = shift;
282 return $self->{_dbh}
283 ||= DBI->connect( $self->{dsn}, $self->{user}, $self->{pass} )
284
285 }
286
287 sub funcid_of_func {
288 my ( $self, $func ) = @_;
289 my $notes = $self->notes;
290 return $notes->{"funcid_of_$func"} if exists $notes->{"funcid_of_$func"};
291
292 my $dbh = $self->dbh;
293 return $notes->{"funcid_of_$func"}
294 = $dbh->selectrow_array(
295 "SELECT funcid FROM funcmap WHERE funcname=?",
296 undef, $func );
297 }
298
299 =head1 COPYRIGHT, LICENSE & WARRANTY
300
301 This software is Copyright 2007, 2008 Six Apart Ltd, [email protected]. All
302 rights reserved.
303
304 TheSchwartz is free software; you may redistribute it and/or modify it
305 under the same terms as Perl itself.
306
307 TheSchwartz comes with no warranty of any kind.
308
309 =cut
310