| 1 |
"""This is a recursive web crawler. Don't go pointing this at random sites; |
| 2 |
it doesn't respect robots.txt and it is pretty brutal about how quickly it |
| 3 |
fetches pages. |
| 4 |
|
| 5 |
The code for this is very short; this is perhaps a good indication |
| 6 |
that this is making the most effective use of the primitves at hand. |
| 7 |
The fetch function does all the work of making http requests, |
| 8 |
searching for new urls, and dispatching new fetches. The GreenPool |
| 9 |
acts as sort of a job coordinator (and concurrency controller of |
| 10 |
course). |
| 11 |
""" |
| 12 |
from eventlet.green.urllib.request import urlopen |
| 13 |
import eventlet |
| 14 |
import re |
| 15 |
|
| 16 |
# http://daringfireball.net/2009/11/liberal_regex_for_matching_urls |
| 17 |
url_regex = re.compile(r'\b(([\w-]+://?|www[.])[^\s()<>]+(?:\([\w\d]+\)|([^[:punct:]\s]|/)))') |
| 18 |
|
| 19 |
|
| 20 |
def fetch(url, seen, pool): |
| 21 |
"""Fetch a url, stick any found urls into the seen set, and |
| 22 |
dispatch any new ones to the pool.""" |
| 23 |
print("fetching", url) |
| 24 |
data = '' |
| 25 |
with eventlet.Timeout(5, False): |
| 26 |
data = urlopen(url).read().decode() |
| 27 |
for url_match in url_regex.finditer(data): |
| 28 |
new_url = url_match.group(0) |
| 29 |
# only send requests to eventlet.net so as not to destroy the internet |
| 30 |
if new_url not in seen and 'eventlet.net' in new_url: |
| 31 |
seen.add(new_url) |
| 32 |
# while this seems stack-recursive, it's actually not: |
| 33 |
# spawned greenthreads start their own stacks |
| 34 |
pool.spawn_n(fetch, new_url, seen, pool) |
| 35 |
|
| 36 |
|
| 37 |
def crawl(start_url): |
| 38 |
"""Recursively crawl starting from *start_url*. Returns a set of |
| 39 |
urls that were found.""" |
| 40 |
pool = eventlet.GreenPool() |
| 41 |
seen = set() |
| 42 |
fetch(start_url, seen, pool) |
| 43 |
pool.waitall() |
| 44 |
return seen |
| 45 |
|
| 46 |
seen = crawl("http://eventlet.net") |
| 47 |
print("I saw these urls:") |
| 48 |
print("\n".join(seen)) |