| 1 |
"""This is a recursive web crawler. Don't go pointing this at random sites; |
| 2 |
it doesn't respect robots.txt and it is pretty brutal about how quickly it |
| 3 |
fetches pages. |
| 4 |
|
| 5 |
This is a kind of "producer/consumer" example; the fetch function produces |
| 6 |
jobs, and the GreenPool itself is the consumer, farming out work concurrently. |
| 7 |
It's easier to write it this way rather than writing a standard consumer loop; |
| 8 |
GreenPool handles any exceptions raised and arranges so that there's a set |
| 9 |
number of "workers", so you don't have to write that tedious management code |
| 10 |
yourself. |
| 11 |
""" |
| 12 |
from eventlet.green.urllib.request import urlopen |
| 13 |
from urllib.request import urlopen |
| 14 |
import eventlet |
| 15 |
import re |
| 16 |
|
| 17 |
# http://daringfireball.net/2009/11/liberal_regex_for_matching_urls |
| 18 |
url_regex = re.compile(r'\b(([\w-]+://?|www[.])[^\s()<>]+(?:\([\w\d]+\)|([^[:punct:]\s]|/)))') |
| 19 |
|
| 20 |
|
| 21 |
def fetch(url, outq): |
| 22 |
"""Fetch a url and push any urls found into a queue.""" |
| 23 |
print("fetching", url) |
| 24 |
data = '' |
| 25 |
with eventlet.Timeout(5, False): |
| 26 |
data = urlopen(url).read().decode() |
| 27 |
for url_match in url_regex.finditer(data): |
| 28 |
new_url = url_match.group(0) |
| 29 |
outq.put(new_url) |
| 30 |
|
| 31 |
|
| 32 |
def producer(start_url): |
| 33 |
"""Recursively crawl starting from *start_url*. Returns a set of |
| 34 |
urls that were found.""" |
| 35 |
pool = eventlet.GreenPool() |
| 36 |
seen = set() |
| 37 |
q = eventlet.Queue() |
| 38 |
q.put(start_url) |
| 39 |
# keep looping if there are new urls, or workers that may produce more urls |
| 40 |
while True: |
| 41 |
while not q.empty(): |
| 42 |
url = q.get() |
| 43 |
# limit requests to eventlet.net so we don't crash all over the internet |
| 44 |
if url not in seen and 'eventlet.net' in url: |
| 45 |
seen.add(url) |
| 46 |
pool.spawn_n(fetch, url, q) |
| 47 |
pool.waitall() |
| 48 |
if q.empty(): |
| 49 |
break |
| 50 |
|
| 51 |
return seen |
| 52 |
|
| 53 |
|
| 54 |
seen = producer("http://eventlet.net") |
| 55 |
print("I saw these urls:") |
| 56 |
print("\n".join(seen)) |