Debian Codegraph @

recipes/miami_herald.recipe

raw
1 __license__ = 'GPL v3'
2 __copyright__ = '2009-2010, Darko Miletic <darko.miletic at gmail.com>'
3 '''
4 miamiherald.com
5 '''
6
7 from calibre.web.feeds.news import BasicNewsRecipe
8
9
10 def classes(classes):
11 q = frozenset(classes.split(' '))
12 return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)})
13
14
15 class TheMiamiHerald(BasicNewsRecipe):
16 title = 'The Miami Herald'
17 __author__ = 'Kovid Goyal'
18 description = "Miami-Dade and Broward's source for the latest breaking local news on sports, weather, business, jobs, real estate, shopping, health, travel, entertainment, & more." # noqa: E501
19 oldest_article = 1
20 max_articles_per_feed = 100
21 publisher = 'The Miami Herald'
22 language = 'en_US'
23 no_stylesheets = True
24 use_embedded_content = False
25 encoding = 'utf-8'
26 remove_javascript = True
27 conversion_options = {'comment': description, 'publisher': publisher, 'language': language}
28
29 keep_only_tags = [classes('story-body')]
30 remove_tags = [classes('social-network-macro social-media')]
31
32 feeds = [
33 ('News', 'https://www.miamiherald.com/news/?widgetName=rssfeed&widgetContentId=712015&getXmlFeed=true'),
34 ('Miami-Dade', 'https://www.miamiherald.com/news/local/community/miami-dade/?widgetName=rssfeed&widgetContentId=712015&getXmlFeed=true'),
35 ('Broward', 'https://www.miamiherald.com/news/local/community/broward/?widgetName=rssfeed&widgetContentId=712015&getXmlFeed=true'),
36 ('Florida Keys', 'https://www.miamiherald.com/news/local/community/florida-keys/?widgetName=rssfeed&widgetContentId=712015&getXmlFeed=true'),
37 ('Florida', 'https://www.miamiherald.com/news/state/florida/?widgetName=rssfeed&widgetContentId=712015&getXmlFeed=true'),
38 ('National', 'https://www.miamiherald.com/news/nation-world/national/?widgetName=rssfeed&widgetContentId=712015&getXmlFeed=true'),
39 ('World', 'https://www.miamiherald.com/news/nation-world/world/?widgetName=rssfeed&widgetContentId=712015&getXmlFeed=true'),
40 ('Americas', 'https://www.miamiherald.com/news/nation-world/world/americas/?widgetName=rssfeed&widgetContentId=712015&getXmlFeed=true'),
41 ('Cuba', 'https://www.miamiherald.com/news/nation-world/world/americas/cuba/?widgetName=rssfeed&widgetContentId=712015&getXmlFeed=true'),
42 ('Haiti', 'https://www.miamiherald.com/news/nation-world/world/americas/haiti/?widgetName=rssfeed&widgetContentId=712015&getXmlFeed=true'),
43 ('Politics', 'https://www.miamiherald.com/news/politics-government/?widgetName=rssfeed&widgetContentId=712015&getXmlFeed=true'),
44 ('Education', 'https://www.miamiherald.com/news/local/education/?widgetName=rssfeed&widgetContentId=712015&getXmlFeed=true'),
45 ('Environment', 'https://www.miamiherald.com/news/local/environment/?widgetName=rssfeed&widgetContentId=712015&getXmlFeed=true'),
46 ('Sports', 'https://www.miamiherald.com/sports/?widgetName=rssfeed&widgetContentId=712015&getXmlFeed=true'),
47 ]
48
49 def get_browser(self, *a, **kw):
50 # MyClatchy servers don't like the user-agent header, they hang forever
51 # when it is present
52 br = BasicNewsRecipe.get_browser(self, *a, **kw)
53 br.addheaders = [x for x in br.addheaders if x[0].lower() != 'user-agent']
54 return br
55
56 def preprocess_html(self, soup):
57 for picture in soup.findAll('picture'):
58 img = picture.find('img')
59 for i, source in enumerate(picture.findAll('source')):
60 if i == 0:
61 img['src'] = source['srcset'].split()[0]
62 source.extract()
63 return soup