Debian Codegraph @

recipes/observer_gb.recipe

raw
1 #!/usr/bin/env python
2 __license__ = 'GPL v3'
3 __copyright__ = '2008, Kovid Goyal [email protected]'
4 __docformat__ = 'restructuredtext en'
5
6 '''
7 http://www.guardian.co.uk/theobserver
8 '''
9 from calibre import strftime
10 from calibre.web.feeds.news import BasicNewsRecipe
11
12
13 class Guardian(BasicNewsRecipe):
14 title = 'The Observer'
15 __author__ = 'jbambridge'
16 language = 'en_GB'
17 simultaneous_downloads = 5
18
19 oldest_article = 7
20 max_articles_per_feed = 100
21 remove_javascript = True
22
23 timefmt = ' [%a, %d %b %Y]'
24
25 filter_regexps = [r'r\.kelkoo\.com']
26
27 keep_only_tags = [
28 dict(
29 name='div',
30 attrs={
31 'id': [
32 'content',
33 'article_header',
34 'main-article-info',
35 ]
36 },
37 ),
38 ]
39 remove_tags = [
40 dict(name='div', attrs={'class': ['video-content', 'videos-third-column']}),
41 dict(
42 name='div',
43 attrs={
44 'id': [
45 'article-toolbox',
46 'subscribe-feeds',
47 ]
48 },
49 ),
50 dict(name='div', attrs={'class': ['promo-component bookshop-books-promo bookshop-books']}),
51 dict(name='ul', attrs={'class': ['pagination']}),
52 dict(name='ul', attrs={'id': ['content-actions']}),
53 dict(name='li', attrs={'id': ['product-image']}),
54 ]
55 use_embedded_content = False
56
57 no_stylesheets = True
58 extra_css = '''
59 .article-attributes{font-size: x-small; font-family:Arial,Helvetica,sans-serif;}
60 .h1{font-size: large ;font-family:georgia,serif; font-weight:bold;}
61 .stand-first-alone{color:#666666; font-size:small; font-family:Arial,Helvetica,sans-serif;}
62 .caption{color:#666666; font-size:x-small; font-family:Arial,Helvetica,sans-serif;}
63 #article-wrapper{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
64 .main-article-info{font-family:Arial,Helvetica,sans-serif;}
65 #full-contents{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
66 #match-stats-summary{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
67 '''
68
69 feeds = [
70 ('Main Section', 'feed://www.guardian.co.uk/theobserver/news/uknews/rss'),
71 ('News', 'feed://www.guardian.co.uk/theobserver/news/rss'),
72 ('World News', 'feed://www.guardian.co.uk/theobserver/news/worldnews/rss'),
73 ('In Focus', 'feed://www.guardian.co.uk/theobserver/news/focus/rss'),
74 ('7 Days', 'feed://www.guardian.co.uk/theobserver/news/7days/rss'),
75 ('Seven Days', 'feed://www.guardian.co.uk/theobserver/news/seven-days/rss'),
76 ('Media', 'feed://www.guardian.co.uk/theobserver/news/media/rss'),
77 ('Business', 'feed://www.guardian.co.uk/theobserver/businessandmedia/rss'),
78 ('Cash', 'feed://www.guardian.co.uk/theobserver/news/cash/rss'),
79 ('Money', 'feed://feeds.guardian.co.uk/theguardian/money/rss'),
80 ('Comment', 'feed://www.guardian.co.uk/theobserver/news/comment/rss'),
81 ('Travel', 'feed://www.guardian.co.uk/theobserver/escape/rss'),
82 ('Culture', 'feed://www.guardian.co.uk/theobserver/review/rss'),
83 ('Money', 'feed://feeds.guardian.co.uk/theguardian/money/rss'),
84 ('TV & Radio', 'feed://www.guardian.co.uk/tv-and-radio/rss'),
85 ('New Review', 'feed://www.guardian.co.uk/theobserver/new-review/rss'),
86 ('Agenda', 'feed://www.guardian.co.uk/theobserver/new-review/agenda/rss'),
87 ('Critics', 'feed://www.guardian.co.uk/theobserver/new-review/critics/rss'),
88 ('Features', 'feed://www.guardian.co.uk/theobserver/new-review/features/rss'),
89 ('Discover', 'feed://www.guardian.co.uk/theobserver/new-review/discover/rss'),
90 ('Books', 'feed://www.guardian.co.uk/theobserver/new-review/books/rss'),
91 ('Magazine', 'feed://www.guardian.co.uk/theobserver/magazine/rss'),
92 ('Regulars', 'feed://www.guardian.co.uk/theobserver/magazine/regulars/rss'),
93 ('Life & Style', 'feed://www.guardian.co.uk/theobserver/magazine/life-and-style/rss'),
94 ('Mag Features', 'feed://www.guardian.co.uk/theobserver/magazine/features2/rss'),
95 ('Sport', 'feed://www.guardian.co.uk/theobserver/sport/rss'),
96 ]
97
98 def get_article_url(self, article):
99 url = article.get('guid', None)
100 if (
101 '/video/' in url
102 or '/flyer/' in url
103 or '/quiz/' in url
104 or '/gallery/' in url
105 or 'ivebeenthere' in url
106 or 'pickthescore' in url
107 or 'audioslideshow' in url
108 ):
109 url = None
110 return url
111
112 def preprocess_html(self, soup):
113
114 for item in soup.findAll(style=True):
115 del item['style']
116
117 for item in soup.findAll(face=True):
118 del item['face']
119 for tag in soup.findAll(name=['ul', 'li']):
120 tag.name = 'div'
121
122 return soup
123
124 def find_sections(self):
125 soup = self.index_to_soup('http://www.guardian.co.uk/theobserver')
126 # find cover pic
127 img = soup.find('img', attrs={'alt': 'Guardian digital edition'})
128 if img is not None:
129 self.cover_url = img['src']
130 # end find cover pic
131
132 idx = soup.find('div', id='book-index')
133 for s in idx.findAll('strong', attrs={'class': 'book'}):
134 a = s.find('a', href=True)
135 yield (self.tag_to_string(a), a['href'])
136
137 def find_articles(self, url):
138 soup = self.index_to_soup(url)
139 div = soup.find('div', attrs={'class': 'book-index'})
140 for ul in div.findAll('ul', attrs={'class': 'trailblock'}):
141 for li in ul.findAll('li'):
142 a = li.find(href=True)
143 if not a:
144 continue
145 title = self.tag_to_string(a)
146 url = a['href']
147 if not title or not url:
148 continue
149 tt = li.find('div', attrs={'class': 'trailtext'})
150 if tt is not None:
151 for da in tt.findAll('a'):
152 da.extract()
153 desc = self.tag_to_string(tt).strip()
154 yield {
155 'title': title,
156 'url': url,
157 'description': desc,
158 'date': strftime('%a, %d %b'),
159 }
160
161 def parse_index(self):
162 try:
163 feeds = []
164 for title, href in self.find_sections():
165 feeds.append((title, list(self.find_articles(href))))
166 return feeds
167 except Exception:
168 raise NotImplementedError
169
170 def postprocess_html(self, soup, first):
171 return soup.findAll('html')[0]