Debian Codegraph @

examples/producer_consumer.py

raw
1 """This is a recursive web crawler. Don't go pointing this at random sites;
2 it doesn't respect robots.txt and it is pretty brutal about how quickly it
3 fetches pages.
4
5 This is a kind of "producer/consumer" example; the fetch function produces
6 jobs, and the GreenPool itself is the consumer, farming out work concurrently.
7 It's easier to write it this way rather than writing a standard consumer loop;
8 GreenPool handles any exceptions raised and arranges so that there's a set
9 number of "workers", so you don't have to write that tedious management code
10 yourself.
11 """
12 from eventlet.green.urllib.request import urlopen
13 from urllib.request import urlopen
14 import eventlet
15 import re
16
17 # http://daringfireball.net/2009/11/liberal_regex_for_matching_urls
18 url_regex = re.compile(r'\b(([\w-]+://?|www[.])[^\s()<>]+(?:\([\w\d]+\)|([^[:punct:]\s]|/)))')
19
20
21 def fetch(url, outq):
22 """Fetch a url and push any urls found into a queue."""
23 print("fetching", url)
24 data = ''
25 with eventlet.Timeout(5, False):
26 data = urlopen(url).read().decode()
27 for url_match in url_regex.finditer(data):
28 new_url = url_match.group(0)
29 outq.put(new_url)
30
31
32 def producer(start_url):
33 """Recursively crawl starting from *start_url*. Returns a set of
34 urls that were found."""
35 pool = eventlet.GreenPool()
36 seen = set()
37 q = eventlet.Queue()
38 q.put(start_url)
39 # keep looping if there are new urls, or workers that may produce more urls
40 while True:
41 while not q.empty():
42 url = q.get()
43 # limit requests to eventlet.net so we don't crash all over the internet
44 if url not in seen and 'eventlet.net' in url:
45 seen.add(url)
46 pool.spawn_n(fetch, url, q)
47 pool.waitall()
48 if q.empty():
49 break
50
51 return seen
52
53
54 seen = producer("http://eventlet.net")
55 print("I saw these urls:")
56 print("\n".join(seen))