Debian Codegraph @

examples/test-range

raw ◀ ▶
Source reconstructed at patch state 0 of 1 — not the installed (fully patched) source.
1 Debian patch touch this file
1 #!/usr/bin/perl -w
2 # Copyright (C) 2002 Nigel Horne <[email protected]>
3 #
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 2 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software
16 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17
18 use strict;
19 use WWW::RobotRules::AnyDBM_File;
20 use LWP::RobotUA; # for spidering and cache lookup
21 use HTTP::Cache::Transparent;
22
23 HTTP::Cache::Transparent::init( { BasePath => '/tmp/cache',
24 Verbose => 1 } );
25
26 my $url = "http://www.bandsman.co.uk";
27 my $rules = WWW::RobotRules::AnyDBM_File->new('www.bandsman.co.uk/Spider', '/tmp/robots.cache');
28
29 my $robot = LWP::RobotUA->new('www.bandsman.co.uk/Spider', '[email protected]', $rules);
30 $robot->timeout(20);
31 # $robot->delay(1/60); # wait 1 second between accesses to same site
32 $robot->delay(0);
33 $robot->protocols_allowed(['http']); # disabling all others
34
35 my $request = new HTTP::Request 'GET' => $url;
36 $request->header('Accept' => 'text/html');
37 $robot->max_size(2048);
38 $request->header('Accept-Encoding' => 'gzip; deflate',
39 'Referer' => 'http://www.bandsman.co.uk');
40
41 my $webdoc = $robot->simple_request($request);
42
43 if(!$webdoc->is_success) {
44 die $webdoc->status_line . "\n";
45 exit 1;
46 }
47 my $content = $webdoc->content;
48
49 if(my $encoding = $webdoc->content_encoding) {
50 if($encoding =~ /gzip/i) {
51 $content = Compress::Zlib::memGunzip($content);
52 if(!defined $content) {
53 die "$url can't be gunziped\n";
54 }
55 } elsif($encoding =~ /deflate/i) {
56 $content = Compress::Zlib::uncompress($content);
57 if(!defined $content) {
58 die "$url can't be uncompressed\n";
59 }
60 }
61 }
62 print "$url: " . length($content) . " bytes\n";
63 print $content . "\n";