Debian Codegraph @

recipes/usatoday.recipe

raw
1 #!/usr/bin/env python
2
3 __license__ = 'GPL v3'
4 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
5 '''
6 usatoday.com
7 '''
8
9 from calibre.web.feeds.news import BasicNewsRecipe
10
11
12 def classes(classes):
13 q = frozenset(classes.split(' '))
14 return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)})
15
16
17 def absurl(url):
18 if url.startswith('/'):
19 return 'https://www.usatoday.com' + url
20
21
22 class USAToday(BasicNewsRecipe):
23 title = 'USA Today'
24 __author__ = 'Kovid Goyal, unkn0wn'
25 description = 'newspaper'
26 encoding = 'utf-8'
27 language = 'en_US'
28
29 use_embedded_content = False
30 timefmt = ' [%d %b %Y]'
31 max_articles_per_feed = 25
32 no_stylesheets = True
33
34 keep_only_tags = [
35 classes('gnt_ar_hl gnt_ar_by gnt_ar_b topper__inner in-depth-content'),
36 ]
37
38 remove_tags = [
39 classes('component--pullquote__icon gnt_ss gnt_em_vp__tp'),
40 dict(attrs={'aria-label': ['advertisement']}),
41 dict(name=['link', 'media-gallery', 'button']),
42 ]
43
44 extra_css = '''
45 h1, h2 {
46 font-size:xx-large;
47 font-family:Arial,Helvetica,sans-serif;}
48 #post-attributes, .info,
49 .clear {
50 font-size:xx-small; color:#4D4D4D;
51 font-family:Arial,Helvetica,sans-serif;
52 }
53 #post-body,
54 #content {
55 font-size:medium;
56 font-family:Arial,Helvetica,sans-serif;
57 }
58 .gnt_em_img_ccw__cap {
59 font-size:small;
60 text-align:center;
61 }
62 '''
63
64 ignore_duplicate_articles = {'title', 'url'}
65 resolve_internal_links = True
66 remove_empty_feeds = True
67
68 def parse_index(self):
69 index = 'https://www.usatoday.com/'
70 sections = [
71 'news',
72 'opinion',
73 'tech',
74 'entertainment',
75 'money',
76 'sports',
77 'travel',
78 'life',
79 'investigations',
80 ]
81 feeds = []
82 soup = self.index_to_soup(index)
83 for sec in sections:
84 section = sec.capitalize()
85 self.log(section)
86 articles = []
87 for a in soup.findAll('a', attrs={'href': lambda x: x and x.startswith('/story/' + sec + '/')}):
88 url = absurl(a['href'].split('?')[0])
89 if url == index + '/story/' + sec + '/':
90 continue
91 title = self.tag_to_string(a)
92 self.log('\t', title, '\n\t\t', url)
93 articles.append({'title': title, 'url': url})
94 if articles:
95 feeds.append((section, articles))
96 return feeds
97
98 def preprocess_html(self, soup):
99 for img in soup.findAll('img', src=True):
100 img['src'] = 'https://www.usatoday.com' + img['src']
101 for div in soup.findAll(attrs={'data-c-caption': True}):
102 div.string = div['data-c-caption']
103 return soup