Debian Codegraph @

recipes/ainonline.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 # License: GPLv3 Copyright: 2019, Jose Ortiz <jlortiz84 at gmail.com>
4
5 from __future__ import absolute_import, division, print_function, unicode_literals
6
7 from pprint import pformat
8
9 from calibre.web.feeds.recipes import BasicNewsRecipe
10
11 INDEX = 'https://www.ainonline.com/'
12
13
14 def absurl(url):
15 if url.startswith('/'):
16 url = INDEX + url[1:]
17 return url
18
19
20 def classes(classes):
21 q = frozenset(classes.split(' '))
22 return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)})
23
24
25 class AINOnline(BasicNewsRecipe):
26 title = 'Aviation International News'
27 __author__ = 'Jose Ortiz'
28 description = (
29 'Aviation International News covers all sectors of the aviation'
30 ' industry, from business aviation to air transport to defense and'
31 ' unmanned aerial vehicles.'
32 )
33 language = 'en'
34 encoding = 'utf-8'
35 no_stylesheets = True
36 remove_javascript = True
37 masthead_url = 'https://www.ainonline.com/sites/ainonline.com/themes/ain30/images/ainlogo-small.jpg'
38 keep_only_tags = [classes('main-content')]
39 remove_tags = [dict(name=['button', 'input']), dict(attrs={'class': lambda x: x and 'comments' in x})]
40
41 def parse_index(self):
42
43 soup = self.index_to_soup(INDEX)
44
45 # css selectors for articles
46 # .view-content [class *= 'featured-story']
47 # .view-content .views-row
48 article_attrs = {'class': lambda x: x and ('featured-story' in x or frozenset(['views-row']).intersection(frozenset(x.split())))}
49
50 ans = []
51
52 for section in soup.findAll(**classes('view-content')):
53 if section.findParent(attrs=dict(id='featured')) is not None:
54 current_section = 'Featured'
55 elif section.findParent(attrs=dict(id='home-top-stories')) is not None:
56 current_section = 'Top Stories'
57 elif section.findParent(attrs=dict(id='quicktabs-container-latest_trending')) is not None:
58 current_section = 'Latest/Trending'
59 else:
60 current_section = 'Articles'
61
62 articles = []
63 for div in section.findAll(attrs=article_attrs):
64 if 'views-row' in div['class']:
65 a = div.find(**classes('title')).a
66 elif 'featured-story' in div['class']:
67 a = div.find(lambda tag: tag.name == 'a' and tag.find(['h1', 'h2', 'h3', 'h4', 'h5', 'h6']) is not None)
68 else:
69 continue
70 title = self.tag_to_string(a)
71 url = absurl(a['href'])
72 desc = ''
73 r = div.find(**classes('teaser'))
74 if r is not None:
75 desc = self.tag_to_string(r)
76 articles.append({'title': title, 'url': url, 'description': desc})
77 if articles:
78 for title, articles_ in ans:
79 if current_section == title:
80 articles_.extend(articles)
81 break
82 else:
83 ans.append((current_section, articles))
84
85 self.log(pformat(ans))
86 return ans