| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
# License: GPLv3 Copyright: 2020, Kovid Goyal <kovid at kovidgoyal.net> |
| 4 |
|
| 5 |
from __future__ import absolute_import, division, print_function, unicode_literals |
| 6 |
|
| 7 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 8 |
|
| 9 |
|
| 10 |
def classes(classes): |
| 11 |
q = frozenset(classes.split(' ')) |
| 12 |
return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)}) |
| 13 |
|
| 14 |
|
| 15 |
class AdvancedUserRecipe1282101454(BasicNewsRecipe): |
| 16 |
title = 'Popular Science' |
| 17 |
language = 'en' |
| 18 |
__author__ = 'Kovid Goyal' |
| 19 |
description = 'Popular Science' |
| 20 |
publisher = 'Popular Science' |
| 21 |
max_articles_per_feed = 100 |
| 22 |
ignore_duplicate_articles = {'url'} |
| 23 |
no_stylesheets = True |
| 24 |
keep_only_tags = [ |
| 25 |
classes('Article-header Article-excerpt Article-author Article-thumbnail Article-bodyText article-title article-dek article-paragraph articlebody'), |
| 26 |
] |
| 27 |
remove_tags = [dict(name='section', attrs={'class': ['recurrent-share']})] |
| 28 |
|
| 29 |
def parse_section_index(self, slug): |
| 30 |
url = 'https://www.popsci.com/{}/'.format(slug) |
| 31 |
self.log('Section:', url) |
| 32 |
soup = self.index_to_soup(url) |
| 33 |
main = soup.find(**classes('category-content-wrapper lg:pb-12')) |
| 34 |
if main is None: |
| 35 |
return |
| 36 |
for div in main.findAll(**classes('card-post')): |
| 37 |
a = div.find('a', href=True, **classes('card-post-image-link')) |
| 38 |
url = a['href'] |
| 39 |
tdiv = div.find(**classes('card-post-title')).find(**classes('mobile')) |
| 40 |
title = self.tag_to_string(tdiv) |
| 41 |
desc = '' |
| 42 |
dek = div.find(**classes('card-post-excerpt')) |
| 43 |
if dek is not None: |
| 44 |
desc = self.tag_to_string(dek) |
| 45 |
self.log(' ', title, url) |
| 46 |
yield {'title': title, 'url': url, 'description': desc} |
| 47 |
|
| 48 |
def parse_index(self): |
| 49 |
sections = [] |
| 50 |
for slug, title in { |
| 51 |
'science': 'Science', |
| 52 |
'technology': 'Technology', |
| 53 |
'diy': 'DIY', |
| 54 |
'reviews': 'Reviews', |
| 55 |
}.items(): |
| 56 |
self.log('slug:', slug) |
| 57 |
articles = list(self.parse_section_index(slug)) |
| 58 |
if articles: |
| 59 |
sections.append((title, articles)) |
| 60 |
return sections |
| 61 |
|
| 62 |
def preprocess_html(self, soup): |
| 63 |
for img in soup.findAll('img', attrs={'data-medsrc': True}): |
| 64 |
img['src'] = img['data-medsrc'] |
| 65 |
return soup |