| 1 |
#!/usr/bin/perl -w |
| 2 |
# Copyright (C) 2002 Nigel Horne <[email protected]> |
| 3 |
# |
| 4 |
# This program is free software; you can redistribute it and/or modify |
| 5 |
# it under the terms of the GNU General Public License as published by |
| 6 |
# the Free Software Foundation; either version 2 of the License, or |
| 7 |
# (at your option) any later version. |
| 8 |
# |
| 9 |
# This program is distributed in the hope that it will be useful, |
| 10 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 11 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 12 |
# GNU General Public License for more details. |
| 13 |
# |
| 14 |
# You should have received a copy of the GNU General Public License |
| 15 |
# along with this program; if not, write to the Free Software |
| 16 |
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. |
| 17 |
|
| 18 |
use strict; |
| 19 |
use WWW::RobotRules::AnyDBM_File; |
| 20 |
use LWP::RobotUA; # for spidering and cache lookup |
| 21 |
use HTTP::Cache::Transparent; |
| 22 |
|
| 23 |
HTTP::Cache::Transparent::init( { BasePath => '/tmp/cache', |
| 24 |
Verbose => 1 } ); |
| 25 |
|
| 26 |
my $url = "http://www.bandsman.co.uk"; |
| 27 |
my $rules = WWW::RobotRules::AnyDBM_File->new('www.bandsman.co.uk/Spider', '/tmp/robots.cache'); |
| 28 |
|
| 29 |
my $robot = LWP::RobotUA->new('www.bandsman.co.uk/Spider', '[email protected]', $rules); |
| 30 |
$robot->timeout(20); |
| 31 |
# $robot->delay(1/60); # wait 1 second between accesses to same site |
| 32 |
$robot->delay(0); |
| 33 |
$robot->protocols_allowed(['http']); # disabling all others |
| 34 |
|
| 35 |
my $request = new HTTP::Request 'GET' => $url; |
| 36 |
$request->header('Accept' => 'text/html'); |
| 37 |
$robot->max_size(2048); |
| 38 |
$request->header('Accept-Encoding' => 'gzip; deflate', |
| 39 |
'Referer' => 'http://www.bandsman.co.uk'); |
| 40 |
|
| 41 |
my $webdoc = $robot->simple_request($request); |
| 42 |
|
| 43 |
if(!$webdoc->is_success) { |
| 44 |
die $webdoc->status_line . "\n"; |
| 45 |
exit 1; |
| 46 |
} |
| 47 |
my $content = $webdoc->content; |
| 48 |
|
| 49 |
if(my $encoding = $webdoc->content_encoding) { |
| 50 |
if($encoding =~ /gzip/i) { |
| 51 |
$content = Compress::Zlib::memGunzip($content); |
| 52 |
if(!defined $content) { |
| 53 |
die "$url can't be gunziped\n"; |
| 54 |
} |
| 55 |
} elsif($encoding =~ /deflate/i) { |
| 56 |
$content = Compress::Zlib::uncompress($content); |
| 57 |
if(!defined $content) { |
| 58 |
die "$url can't be uncompressed\n"; |
| 59 |
} |
| 60 |
} |
| 61 |
} |
| 62 |
print "$url: " . length($content) . " bytes\n"; |
| 63 |
print $content . "\n"; |