| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
""" |
| 4 |
https://www.cirsd.org/en/horizons |
| 5 |
""" |
| 6 |
|
| 7 |
from calibre.web.feeds.news import BasicNewsRecipe, classes |
| 8 |
|
| 9 |
|
| 10 |
class horizons(BasicNewsRecipe): |
| 11 |
title = 'Horizons' |
| 12 |
__author__ = 'unkn0wn' |
| 13 |
description = ( |
| 14 |
' Horizons – Journal of International Relations and Sustainable Development.' |
| 15 |
' Horizons serves as a high-level platform for influential voices from around the world to' |
| 16 |
' provide informed analysis and conduct reasoned exchanges on the full spectrum of issues' |
| 17 |
' that shape international developments.' |
| 18 |
) |
| 19 |
no_stylesheets = True |
| 20 |
use_embedded_content = False |
| 21 |
encoding = 'utf-8' |
| 22 |
language = 'en' |
| 23 |
remove_attributes = ['style', 'height', 'width'] |
| 24 |
masthead_url = 'https://www.cirsd.org/bundles/olpublic/images/horizons-logo.jpg' |
| 25 |
ignore_duplicate_articles = {'url'} |
| 26 |
extra_css = 'em{color:#202020;}' |
| 27 |
simultaneous_downloads = 1 |
| 28 |
|
| 29 |
remove_tags_before = [dict(name='div', **classes('e-con-boxed'))] |
| 30 |
remove_tags_after = [dict(name='div', **classes('e-con-boxed'))] |
| 31 |
|
| 32 |
recipe_specific_options = { |
| 33 |
'issue_url': { |
| 34 |
'short': 'The issue URL ', |
| 35 |
'long': 'For example, https://www.cirsd.org/en/horizons/horizons-winter-2024--issue-no-25', |
| 36 |
} |
| 37 |
} |
| 38 |
|
| 39 |
def preprocess_html(self, soup): |
| 40 |
div = soup.find('div', **classes('e-con-boxed')) |
| 41 |
child = div.find('div', **classes('e-child')) |
| 42 |
if child: |
| 43 |
soup.body.clear() |
| 44 |
soup.body.append(child) |
| 45 |
return soup |
| 46 |
|
| 47 |
def preprocess_raw_html(self, raw, *a): |
| 48 |
return raw.replace('<p> </p>', '').replace('<p dir="ltr"> </p>', '') |
| 49 |
|
| 50 |
def parse_index(self): |
| 51 |
d = self.recipe_specific_options.get('issue_url') |
| 52 |
if d and isinstance(d, str): |
| 53 |
url = d |
| 54 |
else: |
| 55 |
soup = self.index_to_soup('https://www.cirsd.org/en/horizons') |
| 56 |
div = soup.find('div', **classes('horizon-issue')) |
| 57 |
for a in div.findAll('a', href=True): |
| 58 |
if a.find('img'): |
| 59 |
url = a['href'] |
| 60 |
if url.startswith('/'): |
| 61 |
url = 'https://www.cirsd.org' + url |
| 62 |
self.cover_url = a.find('img')['src'] |
| 63 |
self.log(url, '\n', self.cover_url) |
| 64 |
self.title = url.split('/')[-2].replace('-', ' ').title() |
| 65 |
self.log('Downloading Issue: ', self.title) |
| 66 |
|
| 67 |
soup = self.index_to_soup(url) |
| 68 |
toc = soup.find('div', attrs={'class': 'horizon-toc'}) |
| 69 |
feeds = [] |
| 70 |
for section in toc.findAll('div', attrs={'class': 'horizon-section'}): |
| 71 |
secname = self.tag_to_string(section.h3).strip() |
| 72 |
self.log(secname) |
| 73 |
articles = [] |
| 74 |
ul = section.find('ul', attrs={'class': 'horizon-articles-list'}) |
| 75 |
for a in ul.findAll('a', href=True): |
| 76 |
url = a['href'] |
| 77 |
if url.startswith('/'): |
| 78 |
url = 'https://www.cirsd.org' + url |
| 79 |
title = self.tag_to_string(a) |
| 80 |
desc = '' |
| 81 |
span = a.find_next_sibling('span', attrs={'class': 'article-author'}) |
| 82 |
if span: |
| 83 |
desc = self.tag_to_string(span).strip() |
| 84 |
self.log('\t', title, '\n\t', desc, '\n\t\t', url) |
| 85 |
articles.append({'title': title, 'url': url, 'description': desc}) |
| 86 |
if articles: |
| 87 |
feeds.append((secname, articles)) |
| 88 |
return feeds |