| 1 |
#!/usr/bin/env python |
| 2 |
# License: GPLv3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net> |
| 3 |
|
| 4 |
|
| 5 |
import bz2 |
| 6 |
import os |
| 7 |
import ssl |
| 8 |
import sys |
| 9 |
from datetime import UTC, datetime |
| 10 |
from urllib.request import urlopen |
| 11 |
|
| 12 |
|
| 13 |
def download_from_calibre_server(url): |
| 14 |
ca = os.path.join(sys.resources_location, 'calibre-ebook-root-CA.crt') |
| 15 |
ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) |
| 16 |
ssl_context.load_verify_locations(ca) |
| 17 |
return urlopen(url, context=ssl_context).read() |
| 18 |
|
| 19 |
|
| 20 |
def filter_ans(ans): |
| 21 |
return list(filter(None, (x.strip() for x in ans))) |
| 22 |
|
| 23 |
|
| 24 |
def common_user_agents(): |
| 25 |
print('Getting recent UAs...') |
| 26 |
raw = download_from_calibre_server('https://code.calibre-ebook.com/ua-popularity') |
| 27 |
ans = {} |
| 28 |
for line in bz2.decompress(raw).decode('utf-8').splitlines(): |
| 29 |
count, ua = line.partition(':')[::2] |
| 30 |
count = int(count.strip()) |
| 31 |
ua = ua.strip() |
| 32 |
if len(ua) > 25 and 'python' not in ua: |
| 33 |
ans[ua] = count |
| 34 |
return ans, list(sorted(ans, reverse=True, key=ans.__getitem__)) |
| 35 |
|
| 36 |
|
| 37 |
def all_desktop_platforms(user_agents): |
| 38 |
ans = set() |
| 39 |
for ua in user_agents: |
| 40 |
if ' Mobile ' not in ua and 'Mobile/' not in ua and ('Firefox/' in ua or 'Chrome/' in ua): |
| 41 |
plat = ua.partition('(')[2].partition(')')[0] |
| 42 |
parts = plat.split(';') |
| 43 |
if 'Firefox/' in ua: |
| 44 |
del parts[-1] |
| 45 |
ans.add(';'.join(parts)) |
| 46 |
return ans |
| 47 |
|
| 48 |
|
| 49 |
def get_data(): |
| 50 |
ua_freq_map, common = common_user_agents() |
| 51 |
ans = { |
| 52 |
'common_user_agents': common, |
| 53 |
'user_agents_popularity': ua_freq_map, |
| 54 |
'timestamp': datetime.now(UTC).isoformat(), |
| 55 |
} |
| 56 |
ans['desktop_platforms'] = list(all_desktop_platforms(ans['common_user_agents'])) |
| 57 |
return ans |