Debian Codegraph @

recipes/brewiarz.recipe

raw
1 #!/usr/bin/env python
2
3 from __future__ import print_function
4
5 __license__ = 'GPL v3'
6
7 import datetime
8
9 from calibre.web.feeds.news import BasicNewsRecipe
10
11
12 class brewiarz(BasicNewsRecipe):
13 title = 'Brewiarz'
14 __author__ = 'Artur Stachecki <[email protected]>'
15 language = 'pl'
16 description = 'Serwis poświęcony Liturgii Godzin (brewiarzowi) - formie codziennej modlitwy Kościoła katolickiego.'
17 masthead_url = 'http://brewiarz.pl/images/logo2.gif'
18 max_articles_per_feed = 100
19 remove_javascript = True
20 no_stylesheets = True
21 publication_type = 'newspaper'
22 next_days = 1
23
24 def parse_index(self):
25 dec2rom_dict = {
26 '01': 'i',
27 '02': 'ii',
28 '03': 'iii',
29 '04': 'iv',
30 '05': 'v',
31 '06': 'vi',
32 '07': 'vii',
33 '08': 'viii',
34 '09': 'ix',
35 '10': 'x',
36 '11': 'xi',
37 '12': 'xii',
38 }
39
40 weekday_dict = {
41 'Sunday': 'Niedziela',
42 'Monday': 'Poniedziałek',
43 'Tuesday': 'Wtorek',
44 'Wednesday': 'Środa',
45 'Thursday': 'Czwartek',
46 'Friday': 'Piątek',
47 'Saturday': 'Sobota',
48 }
49
50 now = datetime.datetime.now()
51
52 feeds = []
53 for i in range(self.next_days):
54 url_date = now + datetime.timedelta(days=i)
55 url_date_month = url_date.strftime('%m')
56 url_date_month_roman = dec2rom_dict[url_date_month]
57 url_date_day = url_date.strftime('%d')
58 url_date_year = url_date.strftime('%Y')[2:]
59 url_date_weekday = url_date.strftime('%A')
60 url_date_weekday_pl = weekday_dict[url_date_weekday]
61
62 url = 'http://brewiarz.pl/' + url_date_month_roman + '_' + url_date_year + '/' + url_date_day + url_date_month + '/index.php3'
63 articles = self.parse_pages(url)
64 if articles:
65 title = url_date_weekday_pl + ' ' + url_date_day + '.' + url_date_month + '.' + url_date_year
66 feeds.append((title, articles))
67 else:
68 sectors = self.get_sectors(url)
69 for subpage in sectors:
70 title = url_date_weekday_pl + ' ' + url_date_day + '.' + url_date_month + '.' + url_date_year + ' - ' + subpage.string
71 url = 'http://brewiarz.pl/' + url_date_month_roman + '_' + url_date_year + '/' + url_date_day + url_date_month + '/' + subpage['href']
72 print(url)
73 articles = self.parse_pages(url)
74 if articles:
75 feeds.append((title, articles))
76 return feeds
77
78 def get_sectors(self, url):
79 sectors = []
80 soup = self.index_to_soup(url)
81 sectors_table = soup.find(name='table', attrs={'width': '490'})
82 sector_links = sectors_table.findAll(name='a')
83 for sector_links_modified in sector_links:
84 link_parent_text = sector_links_modified.findParent(name='div').text
85 if link_parent_text:
86 sector_links_modified.text = link_parent_text.text
87 sectors.append(sector_links_modified)
88 return sectors
89
90 def parse_pages(self, url):
91 current_articles = []
92 soup = self.index_to_soup(url)
93 www = soup.find(attrs={'class': 'www'})
94 if www:
95 box_title = www.find(text='Teksty LG')
96 article_box_parent = box_title.findParent('ul')
97 article_box_sibling = article_box_parent.findNextSibling('ul')
98 for li in article_box_sibling.findAll('li'):
99 link = li.find(name='a')
100 ol = link.findNextSibling(name='ol')
101 if ol:
102 sublinks = ol.findAll(name='a')
103 for sublink in sublinks:
104 link_title = self.tag_to_string(link) + ' - ' + self.tag_to_string(sublink)
105 link_url_print = sublink['href'].replace('php3', 'php3?kr=_druk&wr=lg&')
106 link_url = url[:-10] + link_url_print
107 current_articles.append({'title': link_title, 'url': link_url, 'description': '', 'date': ''})
108 elif link.findParent(name='ol'):
109 continue
110 else:
111 link_title = self.tag_to_string(link)
112 link_url_print = link['href'].replace('php3', 'php3?kr=_druk&wr=lg&')
113 link_url = url[:-10] + link_url_print
114 current_articles.append({'title': link_title, 'url': link_url, 'description': '', 'date': ''})
115 return current_articles
116 else:
117 return None
118
119 def preprocess_html(self, soup):
120 footer = soup.find(name='a', attrs={'href': 'http://brewiarz.pl'})
121 footer_parent = footer.findParent('div')
122 footer_parent.extract()
123
124 header = soup.find(text='http://brewiarz.pl')
125 header_parent = header.findParent('div')
126 header_parent.extract()
127
128 subheader = soup.find(text='Kolor szat:').findParent('div')
129 subheader.extract()
130
131 color = soup.find('b')
132 color.extract()
133
134 cleaned = self.strip_tags(soup)
135
136 div = cleaned.findAll(name='div')
137 div[1].extract()
138 div[2].extract()
139 div[3].extract()
140
141 return cleaned
142
143 def strip_tags(self, soup_dirty):
144 VALID_TAGS = ['p', 'div', 'br', 'b', 'a', 'title', 'head', 'html', 'body']
145
146 for tag in soup_dirty.findAll(True):
147 if tag.name not in VALID_TAGS:
148 for i, x in enumerate(tag.parent.contents):
149 if x == tag:
150 break
151 else:
152 print("Can't find", tag, 'in', tag.parent)
153 continue
154 for r in reversed(tag.contents):
155 tag.parent.insert(i, r)
156 tag.extract()
157
158 return soup_dirty