| 1 |
#!/usr/bin/perl |
| 2 |
use strict; |
| 3 |
use DBI; |
| 4 |
use Getopt::Long; |
| 5 |
|
| 6 |
my $dbname = "schwartz"; |
| 7 |
my $user = "root"; |
| 8 |
my $pass = ""; |
| 9 |
my $job = ""; |
| 10 |
my $max_age = 0; |
| 11 |
my $max_count = 0; |
| 12 |
|
| 13 |
=head1 NAME |
| 14 |
|
| 15 |
schwartzmon - monitor The Schwartz |
| 16 |
|
| 17 |
=head1 USAGE |
| 18 |
|
| 19 |
Type |
| 20 |
|
| 21 |
schwartzmon --help |
| 22 |
|
| 23 |
to get full usage. |
| 24 |
|
| 25 |
=cut |
| 26 |
|
| 27 |
sub usage { |
| 28 |
die <<USAGE; |
| 29 |
Usage: schwartzmon <command> [OPTS] |
| 30 |
|
| 31 |
Possible commands: |
| 32 |
queues View past-due job queue depths. (default cmd) |
| 33 |
errors View errors. |
| 34 |
|
| 35 |
Global options: |
| 36 |
--job=<JOBNAME> Only look at one specific job name. Else all are considered. |
| 37 |
--user=<user> Connect to the database as this user |
| 38 |
--pass=<pass> Connect to the database with this password |
| 39 |
--database=<db> Connect to this database |
| 40 |
--dsn=<dsn> Connect to the database using this DSN |
| 41 |
|
| 42 |
Options for 'queues' command: |
| 43 |
--maxage=<n> Don't complain if age of overdue job queue is <= 'n' |
| 44 |
--maxcount=<n> Don't complain if depth of overdue job queue is <= 'n' |
| 45 |
|
| 46 |
Options for 'errors' command: |
| 47 |
--follow | -f Like 'tail -f' for tracking the error log |
| 48 |
--last=n Show last 'n' errors from log |
| 49 |
--inlast=n Show errors in last 'n' seconds |
| 50 |
|
| 51 |
Verbosity: |
| 52 |
if no alerts, nothing is printed, and exit status is 0. |
| 53 |
|
| 54 |
Exit status: |
| 55 |
0 if no alerts, |
| 56 |
non-zero if there are alerts, in which case the alerts are printed. |
| 57 |
|
| 58 |
USAGE |
| 59 |
|
| 60 |
} |
| 61 |
|
| 62 |
my $opt_help = 0; |
| 63 |
my ( $opt_follow, $opt_last, $opt_inlast, $opt_func, $dsn ); |
| 64 |
usage() |
| 65 |
unless GetOptions( |
| 66 |
"job=s" => \$job, |
| 67 |
"maxage=i" => \$max_age, |
| 68 |
"maxcount=i" => \$max_count, |
| 69 |
"help" => \$opt_help, |
| 70 |
"follow|f" => \$opt_follow, |
| 71 |
"last=i" => \$opt_last, |
| 72 |
"inlast=i" => \$opt_inlast, |
| 73 |
"user=s" => \$user, |
| 74 |
"pass=s" => \$pass, |
| 75 |
"dsn=s" => \$dsn, |
| 76 |
"database=s" => \$dbname, |
| 77 |
"func=s" => \$opt_func, |
| 78 |
); |
| 79 |
usage() if $opt_help; |
| 80 |
|
| 81 |
my $cmd = shift || "queues"; |
| 82 |
usage() unless $cmd =~ /^queues|errors$/; |
| 83 |
|
| 84 |
my $dbset = DBSet->new; |
| 85 |
|
| 86 |
$dsn ||= "DBI:mysql:$dbname"; |
| 87 |
|
| 88 |
$dbset->add( DBHandle->new( { dsn => $dsn, user => $user, pass => $pass } ) ); |
| 89 |
|
| 90 |
if ( $cmd eq "queues" ) { queues($dbset); } |
| 91 |
if ( $cmd eq "errors" ) { errors($dbset); } |
| 92 |
exit 0; |
| 93 |
|
| 94 |
################# |
| 95 |
|
| 96 |
sub queues { |
| 97 |
my $dbs = shift; |
| 98 |
my $some_alert = 0; |
| 99 |
$dbs->foreach( |
| 100 |
sub { |
| 101 |
my $db = shift; |
| 102 |
my $dbh = $db->dbh or next; |
| 103 |
|
| 104 |
my $funcmap = $dbh->selectall_hashref( |
| 105 |
"SELECT funcid, funcname FROM funcmap", "funcid" ); |
| 106 |
|
| 107 |
foreach my $funcid ( |
| 108 |
sort { $funcmap->{$a}{funcname} cmp $funcmap->{$b}{funcname} } |
| 109 |
keys %$funcmap |
| 110 |
) |
| 111 |
{ |
| 112 |
my $funcname = $funcmap->{$funcid}{funcname}; |
| 113 |
next if $job && $funcname ne $job; |
| 114 |
|
| 115 |
my $now = time(); |
| 116 |
my $inf = $dbh->selectrow_hashref( |
| 117 |
"SELECT COUNT(*) as 'ct', MIN(run_after) 'oldest' FROM job WHERE funcid=? AND run_after <= $now", |
| 118 |
undef, $funcid |
| 119 |
); |
| 120 |
my $behind = $inf->{ct} ? ( $now - $inf->{oldest} ) : 0; |
| 121 |
|
| 122 |
# okay by default, then we apply rules: |
| 123 |
my $okay = 1; |
| 124 |
$okay = 0 if $behind > $max_age; |
| 125 |
$okay = 0 if $inf->{ct} > $max_count; |
| 126 |
next if $okay; |
| 127 |
$some_alert = 1; |
| 128 |
|
| 129 |
print "$funcname\n"; |
| 130 |
print " outstanding: $inf->{ct}\n"; |
| 131 |
print " behind_secs: $behind\n"; |
| 132 |
} |
| 133 |
} |
| 134 |
); |
| 135 |
exit( $some_alert ? 1 : 0 ); |
| 136 |
} |
| 137 |
|
| 138 |
sub errors { |
| 139 |
my $dbs = shift; |
| 140 |
|
| 141 |
if ($opt_follow) { |
| 142 |
follow_errors($dbs); |
| 143 |
} |
| 144 |
|
| 145 |
$opt_last = 100 unless $opt_last || $opt_inlast; |
| 146 |
|
| 147 |
my @rows; |
| 148 |
$dbs->foreach( |
| 149 |
sub { |
| 150 |
my $db = shift; |
| 151 |
my $dbh = $db->dbh |
| 152 |
or next; |
| 153 |
|
| 154 |
my $extra_where = ''; |
| 155 |
if ($opt_func) { |
| 156 |
my $funcid = $db->funcid_of_func($opt_func) || 0; |
| 157 |
$extra_where = "AND funcid=$funcid"; |
| 158 |
} |
| 159 |
|
| 160 |
my $sql; |
| 161 |
if ($opt_last) { |
| 162 |
$sql |
| 163 |
= "SELECT error_time, jobid, message FROM error WHERE 1=1 $extra_where " |
| 164 |
. "ORDER BY error_time DESC LIMIT $opt_last"; |
| 165 |
} |
| 166 |
elsif ($opt_inlast) { |
| 167 |
my $since = time() - $opt_inlast; |
| 168 |
$sql |
| 169 |
= "SELECT error_time, jobid, message FROM error WHERE error_time >= $since $extra_where " |
| 170 |
. "ORDER BY error_time LIMIT 50000"; |
| 171 |
} |
| 172 |
|
| 173 |
my $sth = $dbh->prepare($sql); |
| 174 |
$sth->execute; |
| 175 |
push @rows, $_ while $_ = $sth->fetchrow_hashref; |
| 176 |
} |
| 177 |
); |
| 178 |
|
| 179 |
@rows = sort { $a->{error_time} <=> $b->{error_time} } @rows; |
| 180 |
if ( $opt_last && @rows > $opt_last ) { |
| 181 |
shift @rows while @rows > $opt_last; |
| 182 |
} |
| 183 |
|
| 184 |
foreach my $r (@rows) { |
| 185 |
print_error($r); |
| 186 |
} |
| 187 |
|
| 188 |
} |
| 189 |
|
| 190 |
sub follow_errors { |
| 191 |
my $dbs = shift; |
| 192 |
|
| 193 |
while (1) { |
| 194 |
$dbs->foreach( |
| 195 |
sub { |
| 196 |
my $db = shift; |
| 197 |
my $dbh = $db->dbh |
| 198 |
or next; |
| 199 |
my $notes = $db->notes; |
| 200 |
|
| 201 |
my $lastmax = $notes->{lastmax} || time(); |
| 202 |
my $seen = $notes->{seen} ||= {}; |
| 203 |
|
| 204 |
my $extra_where = ''; |
| 205 |
if ($opt_func) { |
| 206 |
my $funcid = $db->funcid_of_func($opt_func) || 0; |
| 207 |
$extra_where = "AND funcid=$funcid"; |
| 208 |
} |
| 209 |
|
| 210 |
my $sth |
| 211 |
= $dbh->prepare( |
| 212 |
"SELECT error_time, jobid, message FROM error WHERE error_time >= ? $extra_where ORDER BY error_time" |
| 213 |
); |
| 214 |
$sth->execute($lastmax); |
| 215 |
my @errors; |
| 216 |
push @errors, $_ while $_ = $sth->fetchrow_hashref; |
| 217 |
|
| 218 |
my $newmax = $lastmax; |
| 219 |
foreach my $r (@errors) { |
| 220 |
my $sig = join( ",", map { $_, $r->{$_} } sort keys %$r ); |
| 221 |
next if $seen->{$sig}; |
| 222 |
$seen->{$sig} = $r->{error_time}; |
| 223 |
print_error($r); |
| 224 |
$newmax = $r->{error_time} if $r->{error_time} > $newmax; |
| 225 |
} |
| 226 |
|
| 227 |
$notes->{lastmax} = $newmax; |
| 228 |
|
| 229 |
foreach my $sig ( keys %$seen ) { |
| 230 |
my $time = $seen->{$sig}; |
| 231 |
delete $seen->{$sig} if $time < $newmax; |
| 232 |
} |
| 233 |
|
| 234 |
} |
| 235 |
); |
| 236 |
sleep 1; |
| 237 |
} |
| 238 |
} |
| 239 |
|
| 240 |
sub print_error { |
| 241 |
my $r = shift; |
| 242 |
my $msg = $r->{message}; |
| 243 |
$msg =~ s/\s+$//g; |
| 244 |
printf scalar( localtime( $r->{error_time} ) ) . " [$r->{jobid}]: $msg\n"; |
| 245 |
} |
| 246 |
|
| 247 |
package DBSet; |
| 248 |
|
| 249 |
sub new { |
| 250 |
my ( $this, $args ) = @_; |
| 251 |
my $class = ref($this) || $this; |
| 252 |
return bless {}, $class; |
| 253 |
} |
| 254 |
|
| 255 |
sub add { |
| 256 |
my ( $self, $db ) = @_; |
| 257 |
push @$self, $db; |
| 258 |
} |
| 259 |
|
| 260 |
sub foreach { |
| 261 |
my ( $self, $cb ) = @_; |
| 262 |
foreach my $dbh (@$self) { |
| 263 |
$cb->($dbh); |
| 264 |
} |
| 265 |
} |
| 266 |
|
| 267 |
package DBHandle; |
| 268 |
|
| 269 |
sub new { |
| 270 |
my ( $class, $dbinf ) = @_; |
| 271 |
return bless $dbinf, $class; |
| 272 |
} |
| 273 |
|
| 274 |
sub notes { |
| 275 |
my $self = shift; |
| 276 |
return $self->{notes} ||= {}; |
| 277 |
} |
| 278 |
|
| 279 |
# returns DBI handle |
| 280 |
sub dbh { |
| 281 |
my $self = shift; |
| 282 |
return $self->{_dbh} |
| 283 |
||= DBI->connect( $self->{dsn}, $self->{user}, $self->{pass} ) |
| 284 |
|
| 285 |
} |
| 286 |
|
| 287 |
sub funcid_of_func { |
| 288 |
my ( $self, $func ) = @_; |
| 289 |
my $notes = $self->notes; |
| 290 |
return $notes->{"funcid_of_$func"} if exists $notes->{"funcid_of_$func"}; |
| 291 |
|
| 292 |
my $dbh = $self->dbh; |
| 293 |
return $notes->{"funcid_of_$func"} |
| 294 |
= $dbh->selectrow_array( |
| 295 |
"SELECT funcid FROM funcmap WHERE funcname=?", |
| 296 |
undef, $func ); |
| 297 |
} |
| 298 |
|
| 299 |
=head1 COPYRIGHT, LICENSE & WARRANTY |
| 300 |
|
| 301 |
This software is Copyright 2007, 2008 Six Apart Ltd, [email protected]. All |
| 302 |
rights reserved. |
| 303 |
|
| 304 |
TheSchwartz is free software; you may redistribute it and/or modify it |
| 305 |
under the same terms as Perl itself. |
| 306 |
|
| 307 |
TheSchwartz comes with no warranty of any kind. |
| 308 |
|
| 309 |
=cut |
| 310 |
|