Debian Codegraph @

examples/recursive_crawler.py

raw
1 """This is a recursive web crawler. Don't go pointing this at random sites;
2 it doesn't respect robots.txt and it is pretty brutal about how quickly it
3 fetches pages.
4
5 The code for this is very short; this is perhaps a good indication
6 that this is making the most effective use of the primitves at hand.
7 The fetch function does all the work of making http requests,
8 searching for new urls, and dispatching new fetches. The GreenPool
9 acts as sort of a job coordinator (and concurrency controller of
10 course).
11 """
12 from eventlet.green.urllib.request import urlopen
13 import eventlet
14 import re
15
16 # http://daringfireball.net/2009/11/liberal_regex_for_matching_urls
17 url_regex = re.compile(r'\b(([\w-]+://?|www[.])[^\s()<>]+(?:\([\w\d]+\)|([^[:punct:]\s]|/)))')
18
19
20 def fetch(url, seen, pool):
21 """Fetch a url, stick any found urls into the seen set, and
22 dispatch any new ones to the pool."""
23 print("fetching", url)
24 data = ''
25 with eventlet.Timeout(5, False):
26 data = urlopen(url).read().decode()
27 for url_match in url_regex.finditer(data):
28 new_url = url_match.group(0)
29 # only send requests to eventlet.net so as not to destroy the internet
30 if new_url not in seen and 'eventlet.net' in new_url:
31 seen.add(new_url)
32 # while this seems stack-recursive, it's actually not:
33 # spawned greenthreads start their own stacks
34 pool.spawn_n(fetch, new_url, seen, pool)
35
36
37 def crawl(start_url):
38 """Recursively crawl starting from *start_url*. Returns a set of
39 urls that were found."""
40 pool = eventlet.GreenPool()
41 seen = set()
42 fetch(start_url, seen, pool)
43 pool.waitall()
44 return seen
45
46 seen = crawl("http://eventlet.net")
47 print("I saw these urls:")
48 print("\n".join(seen))