| 1 |
#!/usr/bin/env python |
| 2 |
__license__ = 'GPL v3' |
| 3 |
__copyright__ = '2008-2012, Darko Miletic <darko.miletic at gmail.com>' |
| 4 |
''' |
| 5 |
arstechnica.com |
| 6 |
''' |
| 7 |
|
| 8 |
import re |
| 9 |
|
| 10 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 11 |
|
| 12 |
|
| 13 |
def classes(classes): |
| 14 |
q = frozenset(classes.split(' ')) |
| 15 |
return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)}) |
| 16 |
|
| 17 |
|
| 18 |
class ArsTechnica(BasicNewsRecipe): |
| 19 |
title = 'Ars Technica' |
| 20 |
language = 'en' |
| 21 |
__author__ = 'Darko Miletic, Sujata Raman, Alexis Rohou, Tom Sparks' |
| 22 |
description = 'Ars Technica: Serving the technologist for 1.2 decades' |
| 23 |
publisher = 'Conde Nast Publications' |
| 24 |
oldest_article = 5 |
| 25 |
max_articles_per_feed = 100 |
| 26 |
no_stylesheets = True |
| 27 |
encoding = 'utf-8' |
| 28 |
use_embedded_content = False |
| 29 |
remove_empty_feeds = True |
| 30 |
ignore_duplicate_articles = {'url', 'title'} |
| 31 |
masthead_url = 'https://upload.wikimedia.org/wikipedia/commons/5/51/Ars_Technica_logo_%282016%29.svg' |
| 32 |
extra_css = ''' |
| 33 |
body {font-family: Arial,sans-serif} |
| 34 |
.heading{font-family: "Times New Roman",serif} |
| 35 |
.byline{font-weight: bold; line-height: 1em; font-size: 0.625em; text-decoration: none} |
| 36 |
img{display: block} |
| 37 |
.caption-text{font-size:small; font-style:italic} |
| 38 |
.caption-byline{font-size:small; font-style:italic; font-weight:bold} |
| 39 |
.video, .page-numbers, .story-sidebar { display: none } |
| 40 |
.image { display: block } |
| 41 |
''' |
| 42 |
|
| 43 |
recipe_specific_options = { |
| 44 |
'days': { |
| 45 |
'short': 'Oldest article to download from this news source. In days ', |
| 46 |
'long': 'For example, 0.5, gives you articles from the past 12 hours', |
| 47 |
'default': str(oldest_article), |
| 48 |
} |
| 49 |
} |
| 50 |
|
| 51 |
def __init__(self, *args, **kwargs): |
| 52 |
BasicNewsRecipe.__init__(self, *args, **kwargs) |
| 53 |
d = self.recipe_specific_options.get('days') |
| 54 |
if d and isinstance(d, str): |
| 55 |
self.oldest_article = float(d) |
| 56 |
|
| 57 |
keep_only_tags = [ |
| 58 |
dict(itemprop=['article-header', 'article-body']), |
| 59 |
classes('main post-meta article-guts standalone'), |
| 60 |
] |
| 61 |
|
| 62 |
remove_tags = [ |
| 63 |
classes( |
| 64 |
'xf_thread_iframe_wrapper topnav-sections nav-previous nav-next ars-gallery-caption-credit' |
| 65 |
' ars-gallery-caption-text ars-gallery-caption-arrow font-impact sr-only author-mini-bio' |
| 66 |
' taboola-container comments site-header site-footer video corner-info article-expander' |
| 67 |
' left-column related-stories article-social text-settings-dropdown-story intro-image' |
| 68 |
), |
| 69 |
dict(name=['object', 'link', 'embed', 'iframe', 'meta']), |
| 70 |
dict(id=['social-left', 'article-footer-wrap']), |
| 71 |
dict(name='nav', attrs={'class': 'subheading'}), |
| 72 |
] |
| 73 |
remove_attributes = ['lang', 'style'] |
| 74 |
|
| 75 |
# Feed are found here: http://arstechnica.com/rss-feeds/ |
| 76 |
feeds = [ |
| 77 |
('Ars Technica', 'https://feeds.arstechnica.com/arstechnica/index'), |
| 78 |
('Features', 'https://feeds.arstechnica.com/arstechnica/features'), |
| 79 |
('Technology Lab', 'https://feeds.arstechnica.com/arstechnica/technology-lab'), |
| 80 |
('Gear & Gadgets', 'https://feeds.arstechnica.com/arstechnica/gadgets'), |
| 81 |
('Law & Disorder', 'https://feeds.arstechnica.com/arstechnica/tech-policy'), |
| 82 |
# ('Health', 'https://arstechnica.com/health/feed/'), |
| 83 |
# ('Policy', 'https://arstechnica.com/tech-policy/feed/'), |
| 84 |
# ('Security', 'https://arstechnica.com/security/feed/'), |
| 85 |
('Infinite Loop', 'https://feeds.arstechnica.com/arstechnica/apple'), |
| 86 |
('Opposable Thumbs', 'https://feeds.arstechnica.com/arstechnica/gaming'), |
| 87 |
('Scientific Method', 'https://feeds.arstechnica.com/arstechnica/science'), |
| 88 |
('Cars Technica', 'https://feeds.arstechnica.com/arstechnica/cars'), |
| 89 |
('Staff', 'https://feeds.arstechnica.com/arstechnica/staff'), |
| 90 |
# ('AI', 'https://arstechnica.com/staff/ai/feed/'), |
| 91 |
# ('Space', 'https://arstechnica.com/space/feed/'), |
| 92 |
('Ars Cardboard', 'https://feeds.arstechnica.com/arstechnica/cardboard'), |
| 93 |
] |
| 94 |
|
| 95 |
recursions = 1 |
| 96 |
|
| 97 |
def is_link_wanted(self, url, tag): |
| 98 |
return re.search(r'/[0-9]/$', url) is not None |
| 99 |
|
| 100 |
def postprocess_html(self, soup, first_fetch): |
| 101 |
if not first_fetch: |
| 102 |
for x in soup.findAll(itemprop=['headline', 'description']): |
| 103 |
x.extract() |
| 104 |
for x in soup.findAll(**classes('post-meta')): |
| 105 |
x.extract() |
| 106 |
return soup |