Debian Codegraph @

recipes/natgeomag.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 from datetime import date
4 from pprint import pformat
5
6 from calibre.web.feeds.news import BasicNewsRecipe, classes
7
8
9 class NatGeo(BasicNewsRecipe):
10 title = 'National Geographic Magazine'
11 description = 'The National Geographic, an American monthly magazine. Inspiring people to care about the planet since 1888'
12 language = 'en'
13 encoding = 'utf8'
14 publisher = 'nationalgeographic.com'
15 category = 'science, nat geo'
16 __author__ = 'Kovid Goyal, unkn0wn'
17 timefmt = ' [%a, %d %b, %Y]'
18 no_stylesheets = True
19 use_embedded_content = False
20 remove_attributes = ['style']
21 remove_javascript = False
22 masthead_url = 'https://i.natgeofe.com/n/e76f5368-6797-4794-b7f6-8d757c79ea5c/ng-logo-2fl.png?w=600&h=600'
23 remove_empty_feeds = True
24 resolve_internal_links = True
25
26 recipe_specific_options = {
27 'date': {
28 'short': 'The date of the edition to download (Month-YYYY format)',
29 'long': 'For example, March-2023',
30 },
31 'res': {
32 'short': 'For hi-res images, select a resolution from the\nfollowing options: 800, 1000, 1200 or 1500',
33 'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use 400 or 300.',
34 'default': '600',
35 },
36 }
37
38 @property
39 def natgeo_parser(self):
40 ans = getattr(self, '_natgeo_parser', None)
41 if ans is None:
42 from calibre.live import load_module
43
44 self._natgeo_parser = ans = load_module('calibre.web.site_parsers.natgeo')
45 return ans
46
47 def preprocess_raw_html(self, raw_html, url):
48 return self.natgeo_parser.extract_html(raw_html)
49
50 extra_css = '''
51 blockquote { color:#404040; }
52 .byline, i { font-style:italic; color:#202020; }
53 .cap { font-size:small; }
54 img {display:block; margin:0 auto;}
55 .cred { font-style:italic; font-size:small; color:#404040; }
56 .auth, .time, .sub { font-size:small; color:#5c5c5c; }
57 '''
58
59 def parse_index(self):
60 edition = date.today().strftime('%B-%Y')
61 d = self.recipe_specific_options.get('date')
62 if d and isinstance(d, str):
63 edition = d
64 url = 'https://www.nationalgeographic.com/magazine/issue/' + edition.lower()
65 self.log('Downloading ', url)
66 self.timefmt = ' [' + edition + ']'
67 soup = self.index_to_soup(url)
68 # png = re.findall('https://i\.natgeofe\.com\S+?national-geographic-\S+?\.jpg', soup.decode('utf-8'))
69 # self.cover_url = png[0] + '?w=1000&h=1000'
70 self.cover_url = soup.find('meta', attrs={'property': 'og:image'})['content'].split('?')[0] + '?w=1000'
71
72 # self.title = 'National Geographic ' + self.tag_to_string(name)
73 ans = {}
74 if photoart := soup.find(attrs={'class': lambda x: x and 'BgImagePromo__Container__Text__Link' in x.split()}):
75 section = 'Photo Essay'
76 title = self.tag_to_string(photoart)
77 url = photoart['href']
78 if url.startswith('/'):
79 url = 'https://www.nationalgeographic.com' + url
80 articles = ans.setdefault(section, [])
81 articles.append({'title': title, 'url': url})
82 for promo in soup.findAll(**classes('OneUpPromoCard__Content')):
83 if promo.find('a', attrs={'href': True}) and promo.a.get('href'):
84 url = promo.a['href']
85 section = self.tag_to_string(promo.find(**classes('SectionLabel')))
86 title = self.tag_to_string(promo.find(**classes('Card__Content__Heading')))
87 articles = ans.setdefault(section, [])
88 articles.append({'title': title, 'url': url})
89 for gird in soup.findAll(attrs={'class': 'GridPromoTile'}):
90 for article in soup.findAll('article'):
91 a = article.find('a')
92 url = a['href']
93 if url.startswith('/'):
94 url = 'https://www.nationalgeographic.com' + url
95 if '/graphics/' in url:
96 continue
97 section = self.tag_to_string(article.find(**classes('SectionLabel')))
98 title = self.tag_to_string(article.find(**classes('PromoTile__Title--truncated')))
99 articles = ans.setdefault(section, [])
100 articles.append({'title': title, 'url': url})
101 self.log(pformat(ans))
102 return list(ans.items())
103
104 def preprocess_html(self, soup):
105 for h2 in soup.findAll('h2'):
106 h2.name = 'h4'
107 for img in soup.findAll('img', src=True):
108 res = '?w=600'
109 w = self.recipe_specific_options.get('res')
110 if w and isinstance(w, str):
111 res = '?w=' + w
112 img['src'] = img['src'] + res
113 return soup
114
115 def populate_article_metadata(self, article, soup, first):
116 summ = soup.find(attrs={'class': 'byline'})
117 if summ:
118 article.summary = self.tag_to_string(summ)
119 article.text_summary = self.tag_to_string(summ)