Debian Codegraph @

recipes/natgeo.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 from pprint import pformat
4
5 from calibre.web.feeds.news import BasicNewsRecipe, classes
6
7
8 class NatGeo(BasicNewsRecipe):
9 title = 'National Geographic'
10 description = 'News articles from The National Geographic, Download Monthly. Inspiring people to care about the planet since 1888'
11 language = 'en'
12 encoding = 'utf8'
13 publisher = 'nationalgeographic.com'
14 category = 'science, nat geo'
15 __author__ = 'Kovid Goyal, unkn0wn'
16 timefmt = ' [%a, %d %b, %Y]'
17 no_stylesheets = True
18 use_embedded_content = False
19 remove_attributes = ['style']
20 remove_javascript = False
21 masthead_url = 'https://i.natgeofe.com/n/e76f5368-6797-4794-b7f6-8d757c79ea5c/ng-logo-2fl.png?w=600&h=600'
22 remove_empty_feeds = True
23 resolve_internal_links = True
24 ignore_duplicate_articles = {'url'}
25
26 recipe_specific_options = {
27 'res': {
28 'short': 'For hi-res images, select a resolution from the\nfollowing options: 800, 1000, 1200 or 1500',
29 'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use 400 or 300.',
30 'default': '600',
31 }
32 }
33
34 @property
35 def natgeo_parser(self):
36 ans = getattr(self, '_natgeo_parser', None)
37 if ans is None:
38 from calibre.live import load_module
39
40 self._natgeo_parser = ans = load_module('calibre.web.site_parsers.natgeo')
41 return ans
42
43 def preprocess_raw_html(self, raw_html, url):
44 return self.natgeo_parser.extract_html(raw_html)
45
46 extra_css = '''
47 blockquote { color:#404040; }
48 .byline, i { font-style:italic; color:#202020; }
49 .cap { font-size:small; }
50 img {display:block; margin:0 auto;}
51 .cred { font-style:italic; font-size:small; color:#404040; }
52 .auth, .time, .sub { font-size:small; color:#5c5c5c; }
53 '''
54
55 def get_cover_url(self):
56 # soup = self.index_to_soup('https://www.nationalgeographic.com/magazine/')
57 # png = re.findall('https://i\.natgeofe\.com\S+?national-geographic-\S+?\.jpg', soup.decode('utf-8'))
58 from datetime import date
59
60 url = 'https://www.nationalgeographic.com/magazine/issue/' + (date.today().strftime('%B-%Y')).lower()
61 soup = self.index_to_soup(url)
62 png = soup.find('meta', attrs={'property': 'og:image'})['content'].split('?')
63 return png[0] + '?w=1000&h=1000'
64
65 def parse_index(self):
66 pages = [
67 'https://www.nationalgeographic.com/animals',
68 'https://www.nationalgeographic.com/environment',
69 'https://www.nationalgeographic.com/history',
70 'https://www.nationalgeographic.com/science',
71 'https://www.nationalgeographic.com/travel',
72 ]
73
74 feeds = []
75
76 for sec in pages:
77 soup = self.index_to_soup(sec)
78 parsed = self.articles_from_soup(soup)
79 if parsed:
80 feeds += parsed
81 return feeds
82
83 def articles_from_soup(self, soup):
84 ans = {}
85 for article in soup.findAll('article'):
86 a = article.find('a')
87 url = a['href']
88 if url.startswith('/'):
89 url = 'https://www.nationalgeographic.com' + url
90 section = self.tag_to_string(article.find(**classes('SectionLabel')))
91 if section.startswith('Paid Content'):
92 continue
93 title = self.tag_to_string(article.find(**classes('PromoTile__Title--truncated')))
94 articles = ans.setdefault(section, [])
95 articles.append({'title': title, 'url': url})
96 self.log(pformat(ans))
97 return list(ans.items())
98
99 def preprocess_html(self, soup):
100 for h2 in soup.findAll('h2'):
101 h2.name = 'h4'
102 for img in soup.findAll('img', src=True):
103 res = '?w=600'
104 w = self.recipe_specific_options.get('res')
105 if w and isinstance(w, str):
106 res = '?w=' + w
107 img['src'] = img['src'] + res
108 return soup
109
110 def populate_article_metadata(self, article, soup, first):
111 summ = soup.find(attrs={'class': 'byline'})
112 if summ:
113 article.summary = self.tag_to_string(summ)
114 article.text_summary = self.tag_to_string(summ)