| 1 |
#!/usr/bin/env python |
| 2 |
|
| 3 |
from __future__ import print_function |
| 4 |
|
| 5 |
__license__ = 'GPL v3' |
| 6 |
|
| 7 |
import datetime |
| 8 |
|
| 9 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 10 |
|
| 11 |
|
| 12 |
class brewiarz(BasicNewsRecipe): |
| 13 |
title = 'Brewiarz' |
| 14 |
__author__ = 'Artur Stachecki <[email protected]>' |
| 15 |
language = 'pl' |
| 16 |
description = 'Serwis poświęcony Liturgii Godzin (brewiarzowi) - formie codziennej modlitwy Kościoła katolickiego.' |
| 17 |
masthead_url = 'http://brewiarz.pl/images/logo2.gif' |
| 18 |
max_articles_per_feed = 100 |
| 19 |
remove_javascript = True |
| 20 |
no_stylesheets = True |
| 21 |
publication_type = 'newspaper' |
| 22 |
next_days = 1 |
| 23 |
|
| 24 |
def parse_index(self): |
| 25 |
dec2rom_dict = { |
| 26 |
'01': 'i', |
| 27 |
'02': 'ii', |
| 28 |
'03': 'iii', |
| 29 |
'04': 'iv', |
| 30 |
'05': 'v', |
| 31 |
'06': 'vi', |
| 32 |
'07': 'vii', |
| 33 |
'08': 'viii', |
| 34 |
'09': 'ix', |
| 35 |
'10': 'x', |
| 36 |
'11': 'xi', |
| 37 |
'12': 'xii', |
| 38 |
} |
| 39 |
|
| 40 |
weekday_dict = { |
| 41 |
'Sunday': 'Niedziela', |
| 42 |
'Monday': 'Poniedziałek', |
| 43 |
'Tuesday': 'Wtorek', |
| 44 |
'Wednesday': 'Środa', |
| 45 |
'Thursday': 'Czwartek', |
| 46 |
'Friday': 'Piątek', |
| 47 |
'Saturday': 'Sobota', |
| 48 |
} |
| 49 |
|
| 50 |
now = datetime.datetime.now() |
| 51 |
|
| 52 |
feeds = [] |
| 53 |
for i in range(self.next_days): |
| 54 |
url_date = now + datetime.timedelta(days=i) |
| 55 |
url_date_month = url_date.strftime('%m') |
| 56 |
url_date_month_roman = dec2rom_dict[url_date_month] |
| 57 |
url_date_day = url_date.strftime('%d') |
| 58 |
url_date_year = url_date.strftime('%Y')[2:] |
| 59 |
url_date_weekday = url_date.strftime('%A') |
| 60 |
url_date_weekday_pl = weekday_dict[url_date_weekday] |
| 61 |
|
| 62 |
url = 'http://brewiarz.pl/' + url_date_month_roman + '_' + url_date_year + '/' + url_date_day + url_date_month + '/index.php3' |
| 63 |
articles = self.parse_pages(url) |
| 64 |
if articles: |
| 65 |
title = url_date_weekday_pl + ' ' + url_date_day + '.' + url_date_month + '.' + url_date_year |
| 66 |
feeds.append((title, articles)) |
| 67 |
else: |
| 68 |
sectors = self.get_sectors(url) |
| 69 |
for subpage in sectors: |
| 70 |
title = url_date_weekday_pl + ' ' + url_date_day + '.' + url_date_month + '.' + url_date_year + ' - ' + subpage.string |
| 71 |
url = 'http://brewiarz.pl/' + url_date_month_roman + '_' + url_date_year + '/' + url_date_day + url_date_month + '/' + subpage['href'] |
| 72 |
print(url) |
| 73 |
articles = self.parse_pages(url) |
| 74 |
if articles: |
| 75 |
feeds.append((title, articles)) |
| 76 |
return feeds |
| 77 |
|
| 78 |
def get_sectors(self, url): |
| 79 |
sectors = [] |
| 80 |
soup = self.index_to_soup(url) |
| 81 |
sectors_table = soup.find(name='table', attrs={'width': '490'}) |
| 82 |
sector_links = sectors_table.findAll(name='a') |
| 83 |
for sector_links_modified in sector_links: |
| 84 |
link_parent_text = sector_links_modified.findParent(name='div').text |
| 85 |
if link_parent_text: |
| 86 |
sector_links_modified.text = link_parent_text.text |
| 87 |
sectors.append(sector_links_modified) |
| 88 |
return sectors |
| 89 |
|
| 90 |
def parse_pages(self, url): |
| 91 |
current_articles = [] |
| 92 |
soup = self.index_to_soup(url) |
| 93 |
www = soup.find(attrs={'class': 'www'}) |
| 94 |
if www: |
| 95 |
box_title = www.find(text='Teksty LG') |
| 96 |
article_box_parent = box_title.findParent('ul') |
| 97 |
article_box_sibling = article_box_parent.findNextSibling('ul') |
| 98 |
for li in article_box_sibling.findAll('li'): |
| 99 |
link = li.find(name='a') |
| 100 |
ol = link.findNextSibling(name='ol') |
| 101 |
if ol: |
| 102 |
sublinks = ol.findAll(name='a') |
| 103 |
for sublink in sublinks: |
| 104 |
link_title = self.tag_to_string(link) + ' - ' + self.tag_to_string(sublink) |
| 105 |
link_url_print = sublink['href'].replace('php3', 'php3?kr=_druk&wr=lg&') |
| 106 |
link_url = url[:-10] + link_url_print |
| 107 |
current_articles.append({'title': link_title, 'url': link_url, 'description': '', 'date': ''}) |
| 108 |
elif link.findParent(name='ol'): |
| 109 |
continue |
| 110 |
else: |
| 111 |
link_title = self.tag_to_string(link) |
| 112 |
link_url_print = link['href'].replace('php3', 'php3?kr=_druk&wr=lg&') |
| 113 |
link_url = url[:-10] + link_url_print |
| 114 |
current_articles.append({'title': link_title, 'url': link_url, 'description': '', 'date': ''}) |
| 115 |
return current_articles |
| 116 |
else: |
| 117 |
return None |
| 118 |
|
| 119 |
def preprocess_html(self, soup): |
| 120 |
footer = soup.find(name='a', attrs={'href': 'http://brewiarz.pl'}) |
| 121 |
footer_parent = footer.findParent('div') |
| 122 |
footer_parent.extract() |
| 123 |
|
| 124 |
header = soup.find(text='http://brewiarz.pl') |
| 125 |
header_parent = header.findParent('div') |
| 126 |
header_parent.extract() |
| 127 |
|
| 128 |
subheader = soup.find(text='Kolor szat:').findParent('div') |
| 129 |
subheader.extract() |
| 130 |
|
| 131 |
color = soup.find('b') |
| 132 |
color.extract() |
| 133 |
|
| 134 |
cleaned = self.strip_tags(soup) |
| 135 |
|
| 136 |
div = cleaned.findAll(name='div') |
| 137 |
div[1].extract() |
| 138 |
div[2].extract() |
| 139 |
div[3].extract() |
| 140 |
|
| 141 |
return cleaned |
| 142 |
|
| 143 |
def strip_tags(self, soup_dirty): |
| 144 |
VALID_TAGS = ['p', 'div', 'br', 'b', 'a', 'title', 'head', 'html', 'body'] |
| 145 |
|
| 146 |
for tag in soup_dirty.findAll(True): |
| 147 |
if tag.name not in VALID_TAGS: |
| 148 |
for i, x in enumerate(tag.parent.contents): |
| 149 |
if x == tag: |
| 150 |
break |
| 151 |
else: |
| 152 |
print("Can't find", tag, 'in', tag.parent) |
| 153 |
continue |
| 154 |
for r in reversed(tag.contents): |
| 155 |
tag.parent.insert(i, r) |
| 156 |
tag.extract() |
| 157 |
|
| 158 |
return soup_dirty |