Debian Codegraph @

recipes/ald.recipe

raw
1 #!/usr/bin/env python
2 from __future__ import absolute_import, division, print_function, unicode_literals
3
4 __license__ = 'GPL v3'
5 __copyright__ = '2018, PJ Paul'
6 '''
7 Recipe for Arts and Letters Daily website
8 '''
9
10 import re
11 from datetime import date as dt
12 from datetime import timedelta
13 from itertools import compress
14
15 from calibre.utils.date import parse_date
16 from calibre.web.feeds.news import BasicNewsRecipe
17
18
19 class ALD(BasicNewsRecipe):
20 title = 'Arts and Letters Daily'
21 oldest_article = 7
22 max_articles_per_feed = 100
23 auto_cleanup = True
24 recursions = 0
25 ignore_duplicate_articles = {'url'}
26 index = 'https://www.aldaily.com/alt/'
27 cover_url = 'https://www.aldaily.com/static/images/header.gif'
28 __author__ = 'https://github.com/pjpaulpj'
29 language = 'en'
30 encoding = 'utf-8'
31 browser_type = 'qt'
32
33 def parse_index(self):
34 articles_note = []
35 new_books = []
36 essays = []
37 feeds = []
38 soup = self.index_to_soup(self.index)
39 delta = timedelta(days=self.oldest_article)
40 now = dt.today()
41 oldest_date = now - delta
42
43 # Extract a list of dates from the page.
44 # Subset this out to the list of target dates for extraction.
45 date_list = []
46 for div in soup.findAll('div', attrs={'id': 'dayheader'}):
47 date_list.append(self.tag_to_string(div))
48 date_list_clean = [re.sub(r'[^\w]', ' ', date) for date in date_list]
49 date_list_bool = [parse_date(date).date() >= oldest_date for date in date_list_clean]
50 compress_date = list(compress(date_list, date_list_bool))
51
52 # Process each paragraph one by one.
53 # Stop when the text of the previous div is not in the target date list.
54 for div in soup.findAll('div', attrs={'class': 'mobile-front'}):
55 for p in div.findAll('p'):
56 if self.tag_to_string(p.findPreviousSibling('div')) in compress_date:
57 if p.find('a'):
58 title = self.tag_to_string(p)
59 link = p.find('a')['href']
60 if self.tag_to_string(p.findPreviousSibling('h3')) == 'Articles of Note':
61 articles_note.append({'title': title, 'url': link, 'description': '', 'date': ''})
62 elif self.tag_to_string(p.findPreviousSibling('h3')) == 'New Books':
63 new_books.append({'title': title, 'url': link, 'description': '', 'date': ''})
64 else:
65 essays.append({'title': title, 'url': link, 'description': '', 'date': ''})
66 else:
67 break
68 feeds.append(('Articles of Note', articles_note))
69 feeds.append(('New Books', new_books))
70 feeds.append(('Essays and Opinions', essays))
71 return feeds