Debian Codegraph @

recipes/google_news.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 from __future__ import absolute_import, division, print_function, unicode_literals
4
5 import json
6
7 from calibre.ptempfile import PersistentTemporaryFile
8 from calibre.web.feeds.news import BasicNewsRecipe
9
10 # a serarch topic, filled into the string below. You can change that to anything google news should be searched for...
11 terms_to_search_for = (
12 'computer',
13 'books',
14 )
15
16
17 class google_news_de(BasicNewsRecipe):
18 # Title of the Recipe - this is a sample
19 title = 'Google News'
20 cover_url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/da/Google_News_icon.svg/500px-Google_News_icon.svg.png'
21 # Author
22 __author__ = 'Volker Heggemann, VoHe, unkn0wn'
23 # oldest article to download (in days) ---- can be edit by user
24 oldest_article = 1.25
25 # describes itself, ---- can be edit by user
26 max_articles_per_feed = 200
27 # speed up the download on fast computers be careful (I test max.20)
28 # ---- can be edit by user
29 simultaneous_downloads = 10
30 # description, some Reader show this in titlepage
31 description = 'Google News filter by your own recipe. Please read it in calibre software!'
32 # What is the content of?
33 category = 'NEWS'
34 # describes itself, ---- can be edit by user
35 use_embedded_content = False
36 remove_javascript = True
37 # Removes empty feeds - why keep them!?
38 remove_empty_feeds = True
39
40 # remove the rubbish (in ebook)
41 auto_cleanup = True
42
43 articles_are_obfuscated = True
44
45 def get_obfuscated_article(self, url):
46 br = self.get_browser()
47 try:
48 br.open(url)
49 except Exception as e:
50 url = e.hdrs.get('location')
51 soup = self.index_to_soup(url)
52 link = soup.find('a', href=True)
53 skip_sections = [ # add sections you want to skip
54 '/video/',
55 '/videos/',
56 '/media/',
57 'podcast-',
58 ]
59 if any(x in link['href'] for x in skip_sections):
60 self.log('Aborting Article ', link['href'])
61 self.abort_article('skipping video links')
62
63 self.log('Found link: ', link['href'])
64 html = br.open(link['href']).read()
65 pt = PersistentTemporaryFile('.html')
66 pt.write(html)
67 pt.close()
68 return pt.name
69
70 # now the content description and URL follows
71 # feel free to add, wipe out what you need ---- can be edit by user
72 #
73 def get_feeds(self):
74 url = 'https://geolocation-db.com/json'
75 data = self.index_to_soup(url, raw=True)
76 data = json.loads(data)
77 country_code = str(data['country_code']).lower() # for me this is de
78 city = data['city']
79 self.feeds = [
80 ('Google news Topnews for ' + country_code, 'https://news.google.com/news?pz=1&cf=all&ned=' + country_code + '&hl=' + country_code + '&output=rss'),
81 ]
82 if city:
83 location = '{},{}'.format(city, country_code)
84 self.feeds.append(
85 ('Google news for ' + location, 'https://news.google.' + country_code + '/news/rss/headlines/section/geo/' + location),
86 )
87 for searchfor in terms_to_search_for:
88 self.feeds.append((
89 'Google news interested in ' + searchfor,
90 'https://news.google.com/news?cf=all&hl=' + country_code + '+&pz=1&ned=' + country_code + '&q=' + searchfor + '&output=rss',
91 ))
92 return BasicNewsRecipe.get_feeds(self)