Debian Codegraph @

setup/browser_data.py

raw
1 #!/usr/bin/env python
2 # License: GPLv3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net>
3
4
5 import bz2
6 import os
7 import ssl
8 import sys
9 from datetime import UTC, datetime
10 from urllib.request import urlopen
11
12
13 def download_from_calibre_server(url):
14 ca = os.path.join(sys.resources_location, 'calibre-ebook-root-CA.crt')
15 ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
16 ssl_context.load_verify_locations(ca)
17 return urlopen(url, context=ssl_context).read()
18
19
20 def filter_ans(ans):
21 return list(filter(None, (x.strip() for x in ans)))
22
23
24 def common_user_agents():
25 print('Getting recent UAs...')
26 raw = download_from_calibre_server('https://code.calibre-ebook.com/ua-popularity')
27 ans = {}
28 for line in bz2.decompress(raw).decode('utf-8').splitlines():
29 count, ua = line.partition(':')[::2]
30 count = int(count.strip())
31 ua = ua.strip()
32 if len(ua) > 25 and 'python' not in ua:
33 ans[ua] = count
34 return ans, list(sorted(ans, reverse=True, key=ans.__getitem__))
35
36
37 def all_desktop_platforms(user_agents):
38 ans = set()
39 for ua in user_agents:
40 if ' Mobile ' not in ua and 'Mobile/' not in ua and ('Firefox/' in ua or 'Chrome/' in ua):
41 plat = ua.partition('(')[2].partition(')')[0]
42 parts = plat.split(';')
43 if 'Firefox/' in ua:
44 del parts[-1]
45 ans.add(';'.join(parts))
46 return ans
47
48
49 def get_data():
50 ua_freq_map, common = common_user_agents()
51 ans = {
52 'common_user_agents': common,
53 'user_agents_popularity': ua_freq_map,
54 'timestamp': datetime.now(UTC).isoformat(),
55 }
56 ans['desktop_platforms'] = list(all_desktop_platforms(ans['common_user_agents']))
57 return ans