Debian Codegraph @

recipes/the_age.recipe

raw
1 #!/usr/bin/env python
2 __license__ = 'GPL v3'
3 __copyright__ = '2009, Matthew Briggs <[email protected]>'
4 __docformat__ = 'restructuredtext en'
5
6 '''
7 theage.com.au
8 '''
9 import re
10
11 from calibre import strftime
12 from calibre.ebooks.BeautifulSoup import BeautifulSoup
13 from calibre.web.feeds.news import BasicNewsRecipe
14
15
16 class TheAge(BasicNewsRecipe):
17 title = 'The Age'
18 description = 'Business News, World News and Breaking News in Melbourne, Australia'
19 publication_type = 'newspaper'
20 __author__ = 'Matthew Briggs'
21 language = 'en_AU'
22
23 max_articles_per_feed = 1000
24 recursions = 0
25 remove_tags = [dict(name=['table', 'script', 'noscript', 'style']), dict(name='a', attrs={'href': '/'}), dict(name='a', attrs={'href': '/text/'})]
26
27 def get_browser(self):
28 br = BasicNewsRecipe.get_browser(self)
29 br.set_handle_refresh(False)
30 return br
31
32 def parse_index(self):
33
34 soup = BeautifulSoup(self.browser.open('http://www.theage.com.au/text/').read())
35
36 section = None
37 sections = {}
38
39 for tag in soup.findAll(['h3', 'a']):
40 if tag.name == 'h3':
41 section = self.tag_to_string(tag)
42 sections[section] = []
43
44 # Make sure to skip: <a href="/">TheAge</a>
45
46 elif section and tag.get('href'):
47 url = tag['href'].strip()
48 if url.startswith('/'):
49 url = 'http://www.theage.com.au' + url
50 title = self.tag_to_string(tag)
51 if url != 'http://www.theage.com.au':
52 sections[section].append({
53 'title': title,
54 'url': url,
55 'date': strftime('%a, %d %b'),
56 'description': '',
57 'content': '',
58 })
59
60 feeds = []
61
62 # Insert feeds in specified order, if available
63
64 feedSort = ['National', 'World', 'Opinion', 'Columns', 'Business', 'Sport', 'Entertainment']
65 for i in feedSort:
66 if i in sections:
67 feeds.append((i, sections[i]))
68
69 # Done with the sorted feeds
70
71 for i in feedSort:
72 sections.pop(i, None)
73
74 # Append what is left over...
75 for i, s in sections.items():
76 feeds.append((i, s))
77
78 return feeds
79
80 def get_cover_url(self):
81
82 soup = BeautifulSoup(self.browser.open('http://www.theage.com.au/todays-paper').read())
83
84 for i in soup.findAll('a'):
85 href = i['href']
86 if href and re.match(r'http://www.theage.com.au/frontpage/[0-9]+/[0-9]+/[0-9]+/frontpage.pdf', href):
87 return href
88
89 return None
90
91 def preprocess_html(self, soup):
92
93 for p in soup.findAll('p'):
94 # Collapse the paragraph by joining the non-tag contents
95
96 contents = [i for i in p.contents if isinstance(i, type(''))]
97 if contents:
98 contents = ''.join(contents)
99
100 # Filter out what's left of the text-mode navigation stuff
101
102 if re.match(r'((\s)|(\&nbsp\;))*\[[\|\s*]*\]((\s)|(\&nbsp\;))*$', contents):
103 p.extract()
104 continue
105
106 # Shrink the fine print font
107
108 if contents == 'This material is subject to copyright and any unauthorised use, copying or mirroring is prohibited.':
109 p['style'] = 'font-size:small'
110 continue
111
112 return soup