| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
from datetime import date |
| 4 |
from pprint import pformat |
| 5 |
|
| 6 |
from calibre.web.feeds.news import BasicNewsRecipe, classes |
| 7 |
|
| 8 |
|
| 9 |
class NatGeo(BasicNewsRecipe): |
| 10 |
title = 'National Geographic Magazine' |
| 11 |
description = 'The National Geographic, an American monthly magazine. Inspiring people to care about the planet since 1888' |
| 12 |
language = 'en' |
| 13 |
encoding = 'utf8' |
| 14 |
publisher = 'nationalgeographic.com' |
| 15 |
category = 'science, nat geo' |
| 16 |
__author__ = 'Kovid Goyal, unkn0wn' |
| 17 |
timefmt = ' [%a, %d %b, %Y]' |
| 18 |
no_stylesheets = True |
| 19 |
use_embedded_content = False |
| 20 |
remove_attributes = ['style'] |
| 21 |
remove_javascript = False |
| 22 |
masthead_url = 'https://i.natgeofe.com/n/e76f5368-6797-4794-b7f6-8d757c79ea5c/ng-logo-2fl.png?w=600&h=600' |
| 23 |
remove_empty_feeds = True |
| 24 |
resolve_internal_links = True |
| 25 |
|
| 26 |
recipe_specific_options = { |
| 27 |
'date': { |
| 28 |
'short': 'The date of the edition to download (Month-YYYY format)', |
| 29 |
'long': 'For example, March-2023', |
| 30 |
}, |
| 31 |
'res': { |
| 32 |
'short': 'For hi-res images, select a resolution from the\nfollowing options: 800, 1000, 1200 or 1500', |
| 33 |
'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use 400 or 300.', |
| 34 |
'default': '600', |
| 35 |
}, |
| 36 |
} |
| 37 |
|
| 38 |
@property |
| 39 |
def natgeo_parser(self): |
| 40 |
ans = getattr(self, '_natgeo_parser', None) |
| 41 |
if ans is None: |
| 42 |
from calibre.live import load_module |
| 43 |
|
| 44 |
self._natgeo_parser = ans = load_module('calibre.web.site_parsers.natgeo') |
| 45 |
return ans |
| 46 |
|
| 47 |
def preprocess_raw_html(self, raw_html, url): |
| 48 |
return self.natgeo_parser.extract_html(raw_html) |
| 49 |
|
| 50 |
extra_css = ''' |
| 51 |
blockquote { color:#404040; } |
| 52 |
.byline, i { font-style:italic; color:#202020; } |
| 53 |
.cap { font-size:small; } |
| 54 |
img {display:block; margin:0 auto;} |
| 55 |
.cred { font-style:italic; font-size:small; color:#404040; } |
| 56 |
.auth, .time, .sub { font-size:small; color:#5c5c5c; } |
| 57 |
''' |
| 58 |
|
| 59 |
def parse_index(self): |
| 60 |
edition = date.today().strftime('%B-%Y') |
| 61 |
d = self.recipe_specific_options.get('date') |
| 62 |
if d and isinstance(d, str): |
| 63 |
edition = d |
| 64 |
url = 'https://www.nationalgeographic.com/magazine/issue/' + edition.lower() |
| 65 |
self.log('Downloading ', url) |
| 66 |
self.timefmt = ' [' + edition + ']' |
| 67 |
soup = self.index_to_soup(url) |
| 68 |
# png = re.findall('https://i\.natgeofe\.com\S+?national-geographic-\S+?\.jpg', soup.decode('utf-8')) |
| 69 |
# self.cover_url = png[0] + '?w=1000&h=1000' |
| 70 |
self.cover_url = soup.find('meta', attrs={'property': 'og:image'})['content'].split('?')[0] + '?w=1000' |
| 71 |
|
| 72 |
# self.title = 'National Geographic ' + self.tag_to_string(name) |
| 73 |
ans = {} |
| 74 |
if photoart := soup.find(attrs={'class': lambda x: x and 'BgImagePromo__Container__Text__Link' in x.split()}): |
| 75 |
section = 'Photo Essay' |
| 76 |
title = self.tag_to_string(photoart) |
| 77 |
url = photoart['href'] |
| 78 |
if url.startswith('/'): |
| 79 |
url = 'https://www.nationalgeographic.com' + url |
| 80 |
articles = ans.setdefault(section, []) |
| 81 |
articles.append({'title': title, 'url': url}) |
| 82 |
for promo in soup.findAll(**classes('OneUpPromoCard__Content')): |
| 83 |
if promo.find('a', attrs={'href': True}) and promo.a.get('href'): |
| 84 |
url = promo.a['href'] |
| 85 |
section = self.tag_to_string(promo.find(**classes('SectionLabel'))) |
| 86 |
title = self.tag_to_string(promo.find(**classes('Card__Content__Heading'))) |
| 87 |
articles = ans.setdefault(section, []) |
| 88 |
articles.append({'title': title, 'url': url}) |
| 89 |
for gird in soup.findAll(attrs={'class': 'GridPromoTile'}): |
| 90 |
for article in soup.findAll('article'): |
| 91 |
a = article.find('a') |
| 92 |
url = a['href'] |
| 93 |
if url.startswith('/'): |
| 94 |
url = 'https://www.nationalgeographic.com' + url |
| 95 |
if '/graphics/' in url: |
| 96 |
continue |
| 97 |
section = self.tag_to_string(article.find(**classes('SectionLabel'))) |
| 98 |
title = self.tag_to_string(article.find(**classes('PromoTile__Title--truncated'))) |
| 99 |
articles = ans.setdefault(section, []) |
| 100 |
articles.append({'title': title, 'url': url}) |
| 101 |
self.log(pformat(ans)) |
| 102 |
return list(ans.items()) |
| 103 |
|
| 104 |
def preprocess_html(self, soup): |
| 105 |
for h2 in soup.findAll('h2'): |
| 106 |
h2.name = 'h4' |
| 107 |
for img in soup.findAll('img', src=True): |
| 108 |
res = '?w=600' |
| 109 |
w = self.recipe_specific_options.get('res') |
| 110 |
if w and isinstance(w, str): |
| 111 |
res = '?w=' + w |
| 112 |
img['src'] = img['src'] + res |
| 113 |
return soup |
| 114 |
|
| 115 |
def populate_article_metadata(self, article, soup, first): |
| 116 |
summ = soup.find(attrs={'class': 'byline'}) |
| 117 |
if summ: |
| 118 |
article.summary = self.tag_to_string(summ) |
| 119 |
article.text_summary = self.tag_to_string(summ) |