| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
from pprint import pformat |
| 4 |
|
| 5 |
from calibre.web.feeds.news import BasicNewsRecipe, classes |
| 6 |
|
| 7 |
|
| 8 |
class NatGeo(BasicNewsRecipe): |
| 9 |
title = 'National Geographic' |
| 10 |
description = 'News articles from The National Geographic, Download Monthly. Inspiring people to care about the planet since 1888' |
| 11 |
language = 'en' |
| 12 |
encoding = 'utf8' |
| 13 |
publisher = 'nationalgeographic.com' |
| 14 |
category = 'science, nat geo' |
| 15 |
__author__ = 'Kovid Goyal, unkn0wn' |
| 16 |
timefmt = ' [%a, %d %b, %Y]' |
| 17 |
no_stylesheets = True |
| 18 |
use_embedded_content = False |
| 19 |
remove_attributes = ['style'] |
| 20 |
remove_javascript = False |
| 21 |
masthead_url = 'https://i.natgeofe.com/n/e76f5368-6797-4794-b7f6-8d757c79ea5c/ng-logo-2fl.png?w=600&h=600' |
| 22 |
remove_empty_feeds = True |
| 23 |
resolve_internal_links = True |
| 24 |
ignore_duplicate_articles = {'url'} |
| 25 |
|
| 26 |
recipe_specific_options = { |
| 27 |
'res': { |
| 28 |
'short': 'For hi-res images, select a resolution from the\nfollowing options: 800, 1000, 1200 or 1500', |
| 29 |
'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use 400 or 300.', |
| 30 |
'default': '600', |
| 31 |
} |
| 32 |
} |
| 33 |
|
| 34 |
@property |
| 35 |
def natgeo_parser(self): |
| 36 |
ans = getattr(self, '_natgeo_parser', None) |
| 37 |
if ans is None: |
| 38 |
from calibre.live import load_module |
| 39 |
|
| 40 |
self._natgeo_parser = ans = load_module('calibre.web.site_parsers.natgeo') |
| 41 |
return ans |
| 42 |
|
| 43 |
def preprocess_raw_html(self, raw_html, url): |
| 44 |
return self.natgeo_parser.extract_html(raw_html) |
| 45 |
|
| 46 |
extra_css = ''' |
| 47 |
blockquote { color:#404040; } |
| 48 |
.byline, i { font-style:italic; color:#202020; } |
| 49 |
.cap { font-size:small; } |
| 50 |
img {display:block; margin:0 auto;} |
| 51 |
.cred { font-style:italic; font-size:small; color:#404040; } |
| 52 |
.auth, .time, .sub { font-size:small; color:#5c5c5c; } |
| 53 |
''' |
| 54 |
|
| 55 |
def get_cover_url(self): |
| 56 |
# soup = self.index_to_soup('https://www.nationalgeographic.com/magazine/') |
| 57 |
# png = re.findall('https://i\.natgeofe\.com\S+?national-geographic-\S+?\.jpg', soup.decode('utf-8')) |
| 58 |
from datetime import date |
| 59 |
|
| 60 |
url = 'https://www.nationalgeographic.com/magazine/issue/' + (date.today().strftime('%B-%Y')).lower() |
| 61 |
soup = self.index_to_soup(url) |
| 62 |
png = soup.find('meta', attrs={'property': 'og:image'})['content'].split('?') |
| 63 |
return png[0] + '?w=1000&h=1000' |
| 64 |
|
| 65 |
def parse_index(self): |
| 66 |
pages = [ |
| 67 |
'https://www.nationalgeographic.com/animals', |
| 68 |
'https://www.nationalgeographic.com/environment', |
| 69 |
'https://www.nationalgeographic.com/history', |
| 70 |
'https://www.nationalgeographic.com/science', |
| 71 |
'https://www.nationalgeographic.com/travel', |
| 72 |
] |
| 73 |
|
| 74 |
feeds = [] |
| 75 |
|
| 76 |
for sec in pages: |
| 77 |
soup = self.index_to_soup(sec) |
| 78 |
parsed = self.articles_from_soup(soup) |
| 79 |
if parsed: |
| 80 |
feeds += parsed |
| 81 |
return feeds |
| 82 |
|
| 83 |
def articles_from_soup(self, soup): |
| 84 |
ans = {} |
| 85 |
for article in soup.findAll('article'): |
| 86 |
a = article.find('a') |
| 87 |
url = a['href'] |
| 88 |
if url.startswith('/'): |
| 89 |
url = 'https://www.nationalgeographic.com' + url |
| 90 |
section = self.tag_to_string(article.find(**classes('SectionLabel'))) |
| 91 |
if section.startswith('Paid Content'): |
| 92 |
continue |
| 93 |
title = self.tag_to_string(article.find(**classes('PromoTile__Title--truncated'))) |
| 94 |
articles = ans.setdefault(section, []) |
| 95 |
articles.append({'title': title, 'url': url}) |
| 96 |
self.log(pformat(ans)) |
| 97 |
return list(ans.items()) |
| 98 |
|
| 99 |
def preprocess_html(self, soup): |
| 100 |
for h2 in soup.findAll('h2'): |
| 101 |
h2.name = 'h4' |
| 102 |
for img in soup.findAll('img', src=True): |
| 103 |
res = '?w=600' |
| 104 |
w = self.recipe_specific_options.get('res') |
| 105 |
if w and isinstance(w, str): |
| 106 |
res = '?w=' + w |
| 107 |
img['src'] = img['src'] + res |
| 108 |
return soup |
| 109 |
|
| 110 |
def populate_article_metadata(self, article, soup, first): |
| 111 |
summ = soup.find(attrs={'class': 'byline'}) |
| 112 |
if summ: |
| 113 |
article.summary = self.tag_to_string(summ) |
| 114 |
article.text_summary = self.tag_to_string(summ) |