Debian Codegraph @

examples/feedscraper.py

raw
1 """A simple web server that accepts POSTS containing a list of feed urls,
2 and returns the titles of those feeds.
3 """
4 import eventlet
5 feedparser = eventlet.import_patched('feedparser')
6
7 # the pool provides a safety limit on our concurrency
8 pool = eventlet.GreenPool()
9
10
11 def fetch_title(url):
12 d = feedparser.parse(url)
13 return d.feed.get('title', '')
14
15
16 def app(environ, start_response):
17 if environ['REQUEST_METHOD'] != 'POST':
18 start_response('403 Forbidden', [])
19 return []
20
21 # the pile collects the result of a concurrent operation -- in this case,
22 # the collection of feed titles
23 pile = eventlet.GreenPile(pool)
24 for line in environ['wsgi.input'].readlines():
25 url = line.strip()
26 if url:
27 pile.spawn(fetch_title, url)
28 # since the pile is an iterator over the results,
29 # you can use it in all sorts of great Pythonic ways
30 titles = '\n'.join(pile)
31 start_response('200 OK', [('Content-type', 'text/plain')])
32 return [titles]
33
34
35 if __name__ == '__main__':
36 from eventlet import wsgi
37 wsgi.server(eventlet.listen(('localhost', 9010)), app)