Debian Codegraph @

recipes/bbc_brasil.recipe

raw
1 ##
2 # Title: BBC News, Sport, and Blog Calibre Recipe
3 # Contact: mattst - [email protected]
4 ##
5 # License: GNU General Public License v3 - http://www.gnu.org/copyleft/gpl.html
6 # Copyright: mattst - [email protected]
7 ##
8 # Written: November 2011
9 # Last Edited: 2011-11-19
10 ##
11
12 __license__ = 'GNU General Public License v3 - http://www.gnu.org/copyleft/gpl.html'
13 __copyright__ = 'mattst - [email protected]'
14
15
16 '''
17 BBC News, Sport, and Blog Calibre Recipe
18 '''
19
20 # Import the regular expressions module.
21 import re
22
23 # Import the BasicNewsRecipe class which this class extends.
24 from calibre.web.feeds.recipes import BasicNewsRecipe
25
26
27 class BBCBrasilRecipe(BasicNewsRecipe):
28 #
29 # **** IMPORTANT USERS READ ME ****
30 #
31 # First select the feeds you want then scroll down below the feeds list
32 # and select the values you want for the other user preferences, like
33 # oldest_article and such like.
34 #
35 #
36 # Select the BBC rss feeds which you want in your ebook.
37 # Selected feed have NO '#' at their start, de-selected feeds begin with a '#'.
38 #
39 # Eg. ("News Home", "http://feeds.bbci.co.uk/... - include feed.
40 # Eg. #("News Home", "http://feeds.bbci.co.uk/... - do not include feed.
41 #
42 # There are 68 feeds below which constitute the bulk of the available rss
43 # feeds on the BBC web site. These include 5 blogs by editors and
44 # correspondents, 16 sports feeds, 15 'sub' regional feeds (Eg. North West
45 # Wales, Scotland Business), and 7 Welsh language feeds.
46 #
47 # Some of the feeds are low volume (Eg. blogs), or very low volume (Eg. Click)
48 # so if "oldest_article = 1.5" (only articles published in the last 36 hours)
49 # you may get some 'empty feeds' which will not then be included in the ebook.
50 #
51 # The 15 feeds currently selected below are simply my default ones.
52 #
53 # Note: With all 68 feeds selected, oldest_article set to 2,
54 # max_articles_per_feed set to 100, and simultaneous_downloads set to 10,
55 # the ebook creation took 29 minutes on my speedy 100 mbps net connection,
56 # fairly high-end desktop PC running Linux (Ubuntu Lucid-Lynx).
57 # More realistically with 15 feeds selected, oldest_article set to 1.5,
58 # max_articles_per_feed set to 100, and simultaneous_downloads set to 20,
59 # it took 6 minutes. If that's too slow increase 'simultaneous_downloads'.
60 #
61 # Select / de-select the feeds you want in your ebook.
62 #
63 feeds = [
64 ('Primeira P\xe1gina', 'http://www.bbc.co.uk/portuguese/index.xml'),
65 ('\xdaltimas Not\xedcias', 'http://www.bbc.co.uk/portuguese/ultimas_noticias/index.xml'),
66 ('Internacional', 'http://www.bbc.co.uk/portuguese/topicos/internacional/index.xml'),
67 ('Brasil', 'http://www.bbc.co.uk/portuguese/topicos/brasil/index.xml'),
68 ('Am\xe9rica Latina', 'http://www.bbc.co.uk/portuguese/topicos/america_latina/index.xml'),
69 ('Economia', 'http://www.bbc.co.uk/portuguese/topicos/economia/index.xml'),
70 ('Sa\xfade', 'http://www.bbc.co.uk/portuguese/topicos/saude/index.xml'),
71 ('Ci\xeancia e Tecnologia', 'http://www.bbc.co.uk/portuguese/topicos/ciencia_e_tecnologia/index.xml'),
72 ('Cultura', 'http://www.bbc.co.uk/portuguese/topicos/cultura/index.xml'),
73 ('V\xeddeos e Fotos', 'http://www.bbc.co.uk/portuguese/videos_e_fotos/index.xml'),
74 ('Especiais', 'http://www.bbc.co.uk/portuguese/especiais/index.xml'),
75 ]
76
77 # **** SELECT YOUR USER PREFERENCES ****
78
79 # Title to use for the ebook.
80 #
81 title = 'BBC Brasil'
82
83 # A brief description for the ebook.
84 #
85 description = 'Not\xedcias do Brasil e do mundo pela British Broadcasting Corporation'
86
87 # The max number of articles which may be downloaded from each feed.
88 # I've never seen more than about 70 articles in a single feed in the
89 # BBC feeds.
90 #
91 max_articles_per_feed = 100
92
93 # The max age of articles which may be downloaded from each feed. This is
94 # specified in days - note fractions of days are allowed, Eg. 2.5 (2 and a
95 # half days). My default of 1.5 days is the last 36 hours, the point at
96 # which I've decided 'news' becomes 'old news', but be warned this is not
97 # so good for the blogs, technology, magazine, etc., and sports feeds.
98 # You may wish to extend this to 2-5 but watch out ebook creation time will
99 # increase as well. Setting this to 30 will get everything (AFAICT) as long
100 # as max_articles_per_feed remains set high (except for 'Click' which is
101 # v. low volume and its currently oldest article is 4th Feb 2011).
102 #
103 oldest_article = 1.5
104
105 # Number of simultaneous downloads. 20 is consistently working fine on the
106 # BBC News feeds with no problems. Speeds things up from the default of 5.
107 # If you have a lot of feeds and/or have increased oldest_article above 2
108 # then you may wish to try increasing simultaneous_downloads to 25-30,
109 # Or, of course, if you are in a hurry. [I've not tried beyond 20.]
110 #
111 simultaneous_downloads = 20
112
113 # Timeout for fetching files from the server in seconds. The default of
114 # 120 seconds, seems somewhat excessive.
115 #
116 timeout = 30
117
118 # The format string for the date shown on the ebook's first page.
119 # List of all values: http://docs.python.org/library/time.html
120 # Default in news.py has a leading space so that's mirrored here.
121 # As with 'feeds' select/de-select by adding/removing the initial '#',
122 # only one timefmt should be selected, here's a few to choose from.
123 #
124 # [Fri, 14 Nov 2011] (Calibre default)
125 timefmt = ' [%a, %d %b %Y]'
126 # timefmt = ' [%a, %d %b %Y %H:%M]' # [Fri, 14 Nov 2011 18:30]
127 # timefmt = ' [%a, %d %b %Y %I:%M %p]' # [Fri, 14 Nov 2011 06:30 PM]
128 # timefmt = ' [%d %b %Y]' # [14 Nov 2011]
129 # timefmt = ' [%d %b %Y %H:%M]' # [14 Nov 2011 18.30]
130 # timefmt = ' [%Y-%m-%d]' # [2011-11-14]
131 # timefmt = ' [%Y-%m-%d-%H-%M]' # [2011-11-14-18-30]
132
133 #
134 # **** IMPORTANT ****
135 #
136 # DO NOT EDIT BELOW HERE UNLESS YOU KNOW WHAT YOU ARE DOING.
137 #
138 # DO NOT EDIT BELOW HERE UNLESS YOU KNOW WHAT YOU ARE DOING.
139 #
140 # I MEAN IT, YES I DO, ABSOLUTELY, AT YOU OWN RISK. :)
141 #
142 # **** IMPORTANT ****
143 #
144
145 # Author of this recipe.
146 __author__ = 'Carlos Laviola'
147
148 language = 'pt_BR'
149
150 # Set tags.
151 tags = 'news, sport, blog'
152
153 # Set publisher and publication type.
154 publisher = 'BBC'
155 publication_type = 'newspaper'
156
157 # Disable stylesheets from site.
158 no_stylesheets = True
159
160 # Specifies an override encoding for sites that have an incorrect charset
161 # specified. Default of 'None' says to auto-detect. Some other BBC recipes
162 # use 'utf8', which works fine (so use that if necessary) but auto-detecting
163 # with None is working fine, so stick with that for robustness.
164 encoding = None
165
166 # Sets whether a feed has full articles embedded in it. The BBC feeds do
167 # not.
168 use_embedded_content = False
169
170 # Removes empty feeds - why keep them!?
171 remove_empty_feeds = True
172
173 # Create a custom title which fits nicely in the Kindle title list.
174 # Requires "import time" above class declaration, and replacing
175 # title with custom_title in conversion_options (right column only).
176 # Example of string below: "BBC News - 14 Nov 2011"
177 #
178 # custom_title = "BBC News - " + time.strftime('%d %b %Y')
179
180 '''
181 # Conversion options for advanced users, but don't forget to comment out the
182 # current conversion_options below. Avoid setting 'linearize_tables' as that
183 # plays havoc with the 'old style' table based pages.
184 #
185 conversion_options = { 'title' : title,
186 'comments' : description,
187 'tags' : tags,
188 'language' : language,
189 'publisher' : publisher,
190 'authors' : publisher,
191 'smarten_punctuation' : True
192 }
193 '''
194
195 conversion_options = {'smarten_punctuation': True}
196
197 # Specify extra CSS - overrides ALL other CSS (IE. Added last).
198 extra_css = '''body { font-family: verdana, helvetica, sans-serif; }
199 .introduction, .first { font-weight: bold; }
200 .cross-head { font-weight: bold; font-size: 125%; }
201 .cap, .caption { display: block; font-size: 80%; font-style: italic; }
202 .cap, .caption, .caption img, .caption span { display: block; text-align: center; margin: 5px auto; }
203 .byl, .byd, .byline img, .byline-name, .byline-title, .author-name, .author-position,
204 .correspondent-portrait img, .byline-lead-in, .name, .role, .bbc-role { display: block;
205 text-align: center; font-size: 80%; font-style: italic; margin: 1px auto; }
206 .story-date, .published, .datestamp { font-size: 80%; }
207 table { width: 100%; }
208 td img { display: block; margin: 5px auto; }
209 ul { padding-top: 10px; }
210 ol { padding-top: 10px; }
211 li { padding-top: 5px; padding-bottom: 5px; }
212 h1 { text-align: center; font-size: 175%; font-weight: bold; }
213 h2 { text-align: center; font-size: 150%; font-weight: bold; }
214 h3 { text-align: center; font-size: 125%; font-weight: bold; }
215 h4, h5, h6 { text-align: center; font-size: 100%; font-weight: bold; }'''
216
217 # Remove various tag attributes to improve the look of the ebook pages.
218 remove_attributes = ['border', 'cellspacing', 'align', 'cellpadding', 'colspan', 'valign', 'vspace', 'hspace', 'alt', 'width', 'height']
219
220 # Remove the (admittedly rarely used) line breaks, "<br />", which sometimes
221 # cause a section of the ebook to start in an unsightly fashion or, more
222 # frequently, a "<br />" will muck up the formatting of a correspondant's byline.
223 # "<br />" and "<br clear/>" are far more frequently used on the table formatted
224 # style of pages, and really spoil the look of the ebook pages.
225 preprocess_regexps = [(re.compile(r'<br[ ]*/>', re.IGNORECASE), lambda m: ''), (re.compile(r'<br[ ]*clear.*/>', re.IGNORECASE), lambda m: '')]
226
227 # Create regular expressions for tag keeping and removal to make the matches more
228 # robust against minor changes and errors in the HTML, Eg. double spaces, leading
229 # and trailing spaces, missing hyphens, and such like.
230 # Python regular expression ('re' class) page:
231 # http://docs.python.org/library/re.html
232
233 # ***************************************
234 # Regular expressions for keep_only_tags:
235 # ***************************************
236
237 # The BBC News HTML pages use variants of 'storybody' to denote the section of a HTML
238 # page which contains the main text of the article. Match storybody variants: 'storybody',
239 # 'story-body', 'story body','storybody ', etc.
240 storybody_reg_exp = '^.*story[_ -]*body.*$'
241
242 # The BBC sport and 'newsbeat' (features) HTML pages use 'blq_content' to hold the title
243 # and published date. This is one level above the usual news pages which have the title
244 # and date within 'story-body'. This is annoying since 'blq_content' must also be kept,
245 # resulting in a lot of extra things to be removed by remove_tags.
246 blq_content_reg_exp = '^.*blq[_ -]*content.*$'
247
248 # The BBC has an alternative page design structure, which I suspect is an out-of-date
249 # design but which is still used in some articles, Eg. 'Click' (technology), 'FastTrack'
250 # (travel), and in some sport pages. These alternative pages are table based (which is
251 # why I think they are an out-of-date design) and account for -I'm guesstimaking- less
252 # than 1% of all articles. They use a table class 'storycontent' to hold the article
253 # and like blq_content (above) have required lots of extra removal by
254 # remove_tags.
255 story_content_reg_exp = '^.*story[_ -]*content.*$'
256
257 # Keep the sections of the HTML which match the list below. The HTML page created by
258 # Calibre will fill <body> with those sections which are matched. Note that the
259 # blq_content_reg_exp must be listed before storybody_reg_exp in keep_only_tags due to
260 # it being the parent of storybody_reg_exp, that is to say the div class/id 'story-body'
261 # will be inside div class/id 'blq_content' in the HTML (if 'blq_content' is there at
262 # all). If they are the other way around in keep_only_tags then blq_content_reg_exp
263 # will end up being discarded.
264 keep_only_tags = [
265 dict(name='table', attrs={'class': re.compile(story_content_reg_exp, re.IGNORECASE)}),
266 dict(name='div', attrs={'class': re.compile(blq_content_reg_exp, re.IGNORECASE)}),
267 dict(name='div', attrs={'id': re.compile(blq_content_reg_exp, re.IGNORECASE)}),
268 dict(name='div', attrs={'class': re.compile(storybody_reg_exp, re.IGNORECASE)}),
269 dict(name='div', attrs={'id': re.compile(storybody_reg_exp, re.IGNORECASE)}),
270 ]
271
272 # ************************************
273 # Regular expressions for remove_tags:
274 # ************************************
275
276 # Regular expression to remove share-help and variant tags. The share-help class
277 # is used by the site for a variety of 'sharing' type links, Eg. Facebook, delicious,
278 # twitter, email. Removed to avoid page clutter.
279 share_help_reg_exp = '^.*share[_ -]*help.*$'
280
281 # Regular expression to remove embedded-hyper and variant tags. This class is used to
282 # display links to other BBC News articles on the same/similar subject.
283 embedded_hyper_reg_exp = '^.*embed*ed[_ -]*hyper.*$'
284
285 # Regular expression to remove hypertabs and variant tags. This class is used to
286 # display a tab bar at the top of an article which allows the user to switch to
287 # an article (viewed on the same page) providing further info., 'in depth' analysis,
288 # an editorial, a correspondant's blog entry, and such like. The ability to handle
289 # a tab bar of this nature is currently beyond the scope of this recipe and
290 # possibly of Calibre itself (not sure about that - TO DO - check!).
291 hypertabs_reg_exp = '^.*hyper[_ -]*tabs.*$'
292
293 # Regular expression to remove story-feature and variant tags. Eg. 'story-feature',
294 # 'story-feature related narrow', 'story-feature wide', 'story-feature narrow'.
295 # This class is used to add additional info. boxes, or small lists, outside of
296 # the main story. TO DO: Work out a way to incorporate these neatly.
297 story_feature_reg_exp = '^.*story[_ -]*feature.*$'
298
299 # Regular expression to remove video and variant tags, Eg. 'videoInStoryB',
300 # 'videoInStoryC'. This class is used to embed video.
301 video_reg_exp = '^.*video.*$'
302
303 # Regular expression to remove audio and variant tags, Eg. 'audioInStoryD'.
304 # This class is used to embed audio.
305 audio_reg_exp = '^.*audio.*$'
306
307 # Regular expression to remove pictureGallery and variant tags, Eg. 'pictureGallery'.
308 # This class is used to embed a photo slideshow. See also 'slideshow'
309 # below.
310 picture_gallery_reg_exp = '^.*picture.*$'
311
312 # Regular expression to remove slideshow and variant tags, Eg. 'dslideshow-enclosure'.
313 # This class is used to embed a slideshow (not necessarily photo) but both
314 # 'slideshow' and 'pictureGallery' are used for slideshows.
315 slideshow_reg_exp = '^.*slide[_ -]*show.*$'
316
317 # Regular expression to remove social-links and variant tags. This class is used to
318 # display links to a BBC bloggers main page, used in various columnist's blogs
319 # (Eg. Nick Robinson, Robert Preston).
320 social_links_reg_exp = '^.*social[_ -]*links.*$'
321
322 # Regular expression to remove quote and (multi) variant tags, Eg. 'quote',
323 # 'endquote', 'quote-credit', 'quote-credit-title', etc. These are usually
324 # removed by 'story-feature' removal (as they are usually within them), but
325 # not always. The quotation removed is always (AFAICT) in the article text
326 # as well but a 2nd copy is placed in a quote tag to draw attention to it.
327 # The quote class tags may or may not appear in div's.
328 quote_reg_exp = '^.*quote.*$'
329
330 # Regular expression to remove hidden and variant tags, Eg. 'hidden'.
331 # The purpose of these is unclear, they seem to be an internal link to a
332 # section within the article, but the text of the link (Eg. 'Continue reading
333 # the main story') never seems to be displayed anyway. Removed to avoid clutter.
334 # The hidden class tags may or may not appear in div's.
335 hidden_reg_exp = '^.*hidden.*$'
336
337 # Regular expression to remove comment and variant tags, Eg. 'comment-introduction'.
338 # Used on the site to display text about registered users entering
339 # comments.
340 comment_reg_exp = '^.*comment.*$'
341
342 # Regular expression to remove form and variant tags, Eg. 'comment-form'.
343 # Used on the site to allow registered BBC users to fill in forms, typically
344 # for entering comments about an article.
345 form_reg_exp = '^.*form.*$'
346
347 # Extra things to remove due to the addition of 'blq_content' in
348 # keep_only_tags.
349
350 # <div class="story-actions"> Used on sports pages for 'email' and 'print'.
351 story_actions_reg_exp = '^.*story[_ -]*actions.*$'
352
353 # <div class="bookmark-list"> Used on sports pages instead of 'share-help' (for
354 # social networking links).
355 bookmark_list_reg_exp = '^.*bookmark[_ -]*list.*$'
356
357 # <div id="secondary-content" class="content-group">
358 # NOTE: Don't remove class="content-group" that is needed.
359 # Used on sports pages to link to 'similar stories'.
360 secondary_content_reg_exp = '^.*secondary[_ -]*content.*$'
361
362 # <div id="featured-content" class="content-group">
363 # NOTE: Don't remove class="content-group" that is needed.
364 # Used on sports pages to link to pages like 'tables', 'fixtures', etc.
365 featured_content_reg_exp = '^.*featured[_ -]*content.*$'
366
367 # <div id="navigation">
368 # Used on sports pages to link to pages like 'tables', 'fixtures', etc.
369 # Used sometimes instead of "featured-content" above.
370 navigation_reg_exp = '^.*navigation.*$'
371
372 # <a class="skip" href="#blq-container-inner">Skip to top</a>
373 # Used on sports pages to link to the top of the page.
374 skip_reg_exp = '^.*skip.*$'
375
376 # Extra things to remove due to the addition of 'storycontent' in keep_only_tags,
377 # which are the alternative table design based pages. The purpose of some of these
378 # is not entirely clear from the pages (which are a total mess!).
379
380 # Remove mapping based tags, Eg. <map id="world_map">
381 # The dynamic maps don't seem to work during ebook creation. TO DO:
382 # Investigate.
383 map_reg_exp = '^.*map.*$'
384
385 # Remove social bookmarking variation, called 'socialBookMarks'.
386 social_bookmarks_reg_exp = '^.*social[_ -]*bookmarks.*$'
387
388 # Remove page navigation tools, like 'search', 'email', 'print', called
389 # 'blq-mast'.
390 blq_mast_reg_exp = '^.*blq[_ -]*mast.*$'
391
392 # Remove 'sharesb', I think this is a generic 'sharing' class. It seems to appear
393 # alongside 'socialBookMarks' whenever that appears. I am removing it as well
394 # under the assumption that it can appear alone as well.
395 sharesb_reg_exp = '^.*sharesb.*$'
396
397 # Remove class 'o'. The worst named user created css class of all time. The creator
398 # should immediately be fired. I've seen it used to hold nothing at all but with
399 # 20 or so empty lines in it. Also to hold a single link to another article.
400 # Whatever it was designed to do it is not wanted by this recipe. Exact
401 # match only.
402 o_reg_exp = '^o$'
403
404 # Remove 'promotopbg' and 'promobottombg', link lists. Have decided to
405 # use two reg expressions to make removing this (and variants) robust.
406 promo_top_reg_exp = '^.*promotopbg.*$'
407 promo_bottom_reg_exp = '^.*promobottombg.*$'
408
409 # Remove 'nlp', provides heading for link lists. Requires an exact match due to
410 # risk of matching those letters in something needed, unless I see a variation
411 # of 'nlp' used at a later date.
412 nlp_reg_exp = '^nlp$'
413
414 # Remove 'mva', provides embedded floating content of various types. Variant 'mvb'
415 # has also now been seen. Requires an exact match of 'mva' or 'mvb' due to risk of
416 # matching those letters in something needed.
417 mva_or_mvb_reg_exp = '^mv[ab]$'
418
419 # Remove 'mvtb', seems to be page navigation tools, like 'blq-mast'.
420 mvtb_reg_exp = '^mvtb$'
421
422 # Remove 'blq-toplink', class to provide a link to the top of the page.
423 blq_toplink_reg_exp = '^.*blq[_ -]*top[_ -]*link.*$'
424
425 # Remove 'products and services' links, Eg. desktop tools, alerts, and so on.
426 # Eg. Class="servicev4 ukfs_services" - what a mess of a name. Have decided to
427 # use two reg expressions to make removing this (and variants) robust.
428 prods_services_01_reg_exp = '^.*servicev4.*$'
429 prods_services_02_reg_exp = '^.*ukfs[_ -]*services.*$'
430
431 # Remove -what I think is- some kind of navigation tools helper class, though I am
432 # not sure, it's called: 'blq-rst blq-new-nav'. What I do know is it pops up
433 # frequently and it is not wanted. Have decided to use two reg expressions to make
434 # removing this (and variants) robust.
435 blq_misc_01_reg_exp = '^.*blq[_ -]*rst.*$'
436 blq_misc_02_reg_exp = '^.*blq[_ -]*new[_ -]*nav.*$'
437
438 # Remove 'puffbox' - this may only appear inside 'storyextra', so it may not
439 # need removing - I have no clue what it does other than it contains links.
440 # Whatever it is - it is not part of the article and is not wanted.
441 puffbox_reg_exp = '^.*puffbox.*$'
442
443 # Remove 'sibtbg' and 'sibtbgf' - some kind of table formatting classes.
444 sibtbg_reg_exp = '^.*sibtbg.*$'
445
446 # Remove 'storyextra' - links to relevant articles and external sites.
447 storyextra_reg_exp = '^.*story[_ -]*extra.*$'
448
449 remove_tags = [
450 dict(name='div', attrs={'class': re.compile(story_feature_reg_exp, re.IGNORECASE)}),
451 dict(name='div', attrs={'class': re.compile(share_help_reg_exp, re.IGNORECASE)}),
452 dict(name='div', attrs={'class': re.compile(embedded_hyper_reg_exp, re.IGNORECASE)}),
453 dict(name='div', attrs={'class': re.compile(hypertabs_reg_exp, re.IGNORECASE)}),
454 dict(name='div', attrs={'class': re.compile(video_reg_exp, re.IGNORECASE)}),
455 dict(name='div', attrs={'class': re.compile(audio_reg_exp, re.IGNORECASE)}),
456 dict(name='div', attrs={'class': re.compile(picture_gallery_reg_exp, re.IGNORECASE)}),
457 dict(name='div', attrs={'class': re.compile(slideshow_reg_exp, re.IGNORECASE)}),
458 dict(name='div', attrs={'class': re.compile(quote_reg_exp, re.IGNORECASE)}),
459 dict(name='div', attrs={'class': re.compile(hidden_reg_exp, re.IGNORECASE)}),
460 dict(name='div', attrs={'class': re.compile(comment_reg_exp, re.IGNORECASE)}),
461 dict(name='div', attrs={'class': re.compile(story_actions_reg_exp, re.IGNORECASE)}),
462 dict(name='div', attrs={'class': re.compile(bookmark_list_reg_exp, re.IGNORECASE)}),
463 dict(name='div', attrs={'id': re.compile(secondary_content_reg_exp, re.IGNORECASE)}),
464 dict(name='div', attrs={'id': re.compile(featured_content_reg_exp, re.IGNORECASE)}),
465 dict(name='div', attrs={'id': re.compile(navigation_reg_exp, re.IGNORECASE)}),
466 dict(name='form', attrs={'id': re.compile(form_reg_exp, re.IGNORECASE)}),
467 dict(attrs={'class': re.compile(quote_reg_exp, re.IGNORECASE)}),
468 dict(attrs={'class': re.compile(hidden_reg_exp, re.IGNORECASE)}),
469 dict(attrs={'class': re.compile(social_links_reg_exp, re.IGNORECASE)}),
470 dict(attrs={'class': re.compile(comment_reg_exp, re.IGNORECASE)}),
471 dict(attrs={'class': re.compile(skip_reg_exp, re.IGNORECASE)}),
472 dict(name='map', attrs={'id': re.compile(map_reg_exp, re.IGNORECASE)}),
473 dict(name='map', attrs={'name': re.compile(map_reg_exp, re.IGNORECASE)}),
474 dict(name='div', attrs={'id': re.compile(social_bookmarks_reg_exp, re.IGNORECASE)}),
475 dict(name='div', attrs={'id': re.compile(blq_mast_reg_exp, re.IGNORECASE)}),
476 dict(name='div', attrs={'class': re.compile(sharesb_reg_exp, re.IGNORECASE)}),
477 dict(name='div', attrs={'class': re.compile(o_reg_exp, re.IGNORECASE)}),
478 dict(name='div', attrs={'class': re.compile(promo_top_reg_exp, re.IGNORECASE)}),
479 dict(name='div', attrs={'class': re.compile(promo_bottom_reg_exp, re.IGNORECASE)}),
480 dict(name='div', attrs={'class': re.compile(nlp_reg_exp, re.IGNORECASE)}),
481 dict(name='div', attrs={'class': re.compile(mva_or_mvb_reg_exp, re.IGNORECASE)}),
482 dict(name='div', attrs={'class': re.compile(mvtb_reg_exp, re.IGNORECASE)}),
483 dict(name='div', attrs={'class': re.compile(blq_toplink_reg_exp, re.IGNORECASE)}),
484 dict(name='div', attrs={'class': re.compile(prods_services_01_reg_exp, re.IGNORECASE)}),
485 dict(name='div', attrs={'class': re.compile(prods_services_02_reg_exp, re.IGNORECASE)}),
486 dict(name='div', attrs={'class': re.compile(blq_misc_01_reg_exp, re.IGNORECASE)}),
487 dict(name='div', attrs={'class': re.compile(blq_misc_02_reg_exp, re.IGNORECASE)}),
488 dict(name='div', attrs={'class': re.compile(puffbox_reg_exp, re.IGNORECASE)}),
489 dict(attrs={'class': re.compile(sibtbg_reg_exp, re.IGNORECASE)}),
490 dict(attrs={'class': re.compile(storyextra_reg_exp, re.IGNORECASE)}),
491 dict(name='div', attrs={'class': 'tools-container'}),
492 dict(name='div', attrs={'class': 'tools-container-end'}),
493 dict(name='div', attrs={'class': 'g-block story-body contextual-links'}),
494 dict(name='div', attrs={'class': ' g-w11 sidebar'}),
495 ]
496
497 # Uses url to create and return the 'printer friendly' version of the url.
498 # In other words the 'print this page' address of the page.
499 #
500 # There are 3 types of urls used in the BBC site's rss feeds. There is just
501 # 1 type for the standard news while there are 2 used for sports feed urls.
502 # Note: Sports urls are linked from regular news feeds (Eg. 'News Home') when
503 # there is a major story of interest to 'everyone'. So even if no BBC sports
504 # feeds are added to 'feeds' the logic of this method is still needed to avoid
505 # blank / missing / empty articles which have an index title and then no
506 # body.
507 def print_version(self, url):
508
509 # Handle sports page urls type 01:
510 if url.find('go/rss/-/sport1/') != -1:
511 temp_url = url.replace('go/rss/-/', '')
512
513 # Handle sports page urls type 02:
514 elif url.find('go/rss/int/news/-/sport1/') != -1:
515 temp_url = url.replace('go/rss/int/news/-/', '')
516
517 # Handle regular news page urls:
518 else:
519 temp_url = url.replace('go/rss/int/news/-/', '')
520
521 # Always add "?print=true" to the end of the url.
522 print_url = temp_url + '?print=true'
523
524 return print_url
525
526 # Remove articles in feeds based on a string in the article title or url.
527 #
528 # Code logic written by: Starson17 - posted in: "Recipes - Re-usable code"
529 # thread, in post with title: "Remove articles from feed", see url:
530 # http://www.mobileread.com/forums/showpost.php?p=1165462&postcount=6
531 # Many thanks and all credit to Starson17.
532 #
533 # Starson17's code has obviously been altered to suite my requirements.
534 def parse_feeds(self):
535
536 # Call parent's method.
537 feeds = BasicNewsRecipe.parse_feeds(self)
538
539 # Loop through all feeds.
540 for feed in feeds:
541 # Loop through all articles in feed.
542 for article in feed.articles[:]:
543 # Match key words and remove article if there's a match.
544
545 # Most BBC rss feed video only 'articles' use upper case 'VIDEO'
546 # as a title prefix. Just match upper case 'VIDEO', so that
547 # articles like 'Video game banned' won't be matched and
548 # removed.
549 if 'VIDEO' in article.title:
550 feed.articles.remove(article)
551
552 # Most BBC rss feed audio only 'articles' use upper case 'AUDIO'
553 # as a title prefix. Just match upper case 'AUDIO', so that
554 # articles like 'Hi-Def audio...' won't be matched and removed.
555 elif 'AUDIO' in article.title:
556 feed.articles.remove(article)
557
558 # Most BBC rss feed photo slideshow 'articles' use 'In Pictures',
559 # 'In pictures', and 'in pictures', somewhere in their title.
560 # Match any case of that phrase.
561 elif 'IN PICTURES' in article.title.upper():
562 feed.articles.remove(article)
563
564 # As above, but user contributed pictures. Match any case.
565 elif 'YOUR PICTURES' in article.title.upper():
566 feed.articles.remove(article)
567
568 # 'Sportsday Live' are articles which contain a constantly and
569 # dynamically updated 'running commentary' during a live sporting
570 # event. Match any case.
571 elif 'SPORTSDAY LIVE' in article.title.upper():
572 feed.articles.remove(article)
573
574 # Sometimes 'Sportsday Live' (above) becomes 'Live - Sport Name'.
575 # These are being matched below using 'Live - ' because removing all
576 # articles with 'live' in their titles would remove some articles
577 # that are in fact not live sports pages. Match any case.
578 elif 'LIVE - ' in article.title.upper():
579 feed.articles.remove(article)
580
581 # 'Quiz of the week' is a Flash player weekly news quiz. Match only
582 # the 'Quiz of the' part in anticipation of monthly and yearly
583 # variants. Match any case.
584 elif 'QUIZ OF THE' in article.title.upper():
585 feed.articles.remove(article)
586
587 # Remove articles with 'scorecards' in the url. These are BBC sports
588 # pages which just display a cricket scorecard. The pages have a mass
589 # of table and css entries to display the scorecards nicely. Probably
590 # could make them work with this recipe, but might take a whole day
591 # of work to sort out all the css - basically a formatting
592 # nightmare.
593 elif 'scorecards' in article.url:
594 feed.articles.remove(article)
595
596 return feeds
597
598
599 # End of class and file.