| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
from __future__ import absolute_import, division, print_function, unicode_literals |
| 4 |
|
| 5 |
import json |
| 6 |
|
| 7 |
from calibre.ptempfile import PersistentTemporaryFile |
| 8 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 9 |
|
| 10 |
# a serarch topic, filled into the string below. You can change that to anything google news should be searched for... |
| 11 |
terms_to_search_for = ( |
| 12 |
'computer', |
| 13 |
'books', |
| 14 |
) |
| 15 |
|
| 16 |
|
| 17 |
class google_news_de(BasicNewsRecipe): |
| 18 |
# Title of the Recipe - this is a sample |
| 19 |
title = 'Google News' |
| 20 |
cover_url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/da/Google_News_icon.svg/500px-Google_News_icon.svg.png' |
| 21 |
# Author |
| 22 |
__author__ = 'Volker Heggemann, VoHe, unkn0wn' |
| 23 |
# oldest article to download (in days) ---- can be edit by user |
| 24 |
oldest_article = 1.25 |
| 25 |
# describes itself, ---- can be edit by user |
| 26 |
max_articles_per_feed = 200 |
| 27 |
# speed up the download on fast computers be careful (I test max.20) |
| 28 |
# ---- can be edit by user |
| 29 |
simultaneous_downloads = 10 |
| 30 |
# description, some Reader show this in titlepage |
| 31 |
description = 'Google News filter by your own recipe. Please read it in calibre software!' |
| 32 |
# What is the content of? |
| 33 |
category = 'NEWS' |
| 34 |
# describes itself, ---- can be edit by user |
| 35 |
use_embedded_content = False |
| 36 |
remove_javascript = True |
| 37 |
# Removes empty feeds - why keep them!? |
| 38 |
remove_empty_feeds = True |
| 39 |
|
| 40 |
# remove the rubbish (in ebook) |
| 41 |
auto_cleanup = True |
| 42 |
|
| 43 |
articles_are_obfuscated = True |
| 44 |
|
| 45 |
def get_obfuscated_article(self, url): |
| 46 |
br = self.get_browser() |
| 47 |
try: |
| 48 |
br.open(url) |
| 49 |
except Exception as e: |
| 50 |
url = e.hdrs.get('location') |
| 51 |
soup = self.index_to_soup(url) |
| 52 |
link = soup.find('a', href=True) |
| 53 |
skip_sections = [ # add sections you want to skip |
| 54 |
'/video/', |
| 55 |
'/videos/', |
| 56 |
'/media/', |
| 57 |
'podcast-', |
| 58 |
] |
| 59 |
if any(x in link['href'] for x in skip_sections): |
| 60 |
self.log('Aborting Article ', link['href']) |
| 61 |
self.abort_article('skipping video links') |
| 62 |
|
| 63 |
self.log('Found link: ', link['href']) |
| 64 |
html = br.open(link['href']).read() |
| 65 |
pt = PersistentTemporaryFile('.html') |
| 66 |
pt.write(html) |
| 67 |
pt.close() |
| 68 |
return pt.name |
| 69 |
|
| 70 |
# now the content description and URL follows |
| 71 |
# feel free to add, wipe out what you need ---- can be edit by user |
| 72 |
# |
| 73 |
def get_feeds(self): |
| 74 |
url = 'https://geolocation-db.com/json' |
| 75 |
data = self.index_to_soup(url, raw=True) |
| 76 |
data = json.loads(data) |
| 77 |
country_code = str(data['country_code']).lower() # for me this is de |
| 78 |
city = data['city'] |
| 79 |
self.feeds = [ |
| 80 |
('Google news Topnews for ' + country_code, 'https://news.google.com/news?pz=1&cf=all&ned=' + country_code + '&hl=' + country_code + '&output=rss'), |
| 81 |
] |
| 82 |
if city: |
| 83 |
location = '{},{}'.format(city, country_code) |
| 84 |
self.feeds.append( |
| 85 |
('Google news for ' + location, 'https://news.google.' + country_code + '/news/rss/headlines/section/geo/' + location), |
| 86 |
) |
| 87 |
for searchfor in terms_to_search_for: |
| 88 |
self.feeds.append(( |
| 89 |
'Google news interested in ' + searchfor, |
| 90 |
'https://news.google.com/news?cf=all&hl=' + country_code + '+&pz=1&ned=' + country_code + '&q=' + searchfor + '&output=rss', |
| 91 |
)) |
| 92 |
return BasicNewsRecipe.get_feeds(self) |