| 1 |
#!/usr/bin/python3 |
| 2 |
|
| 3 |
import binascii |
| 4 |
import json |
| 5 |
import logging |
| 6 |
import re |
| 7 |
import sys |
| 8 |
from collections import defaultdict |
| 9 |
|
| 10 |
MASK_MAGIC_REGEX = re.compile(r'[*?!@$]') |
| 11 |
|
| 12 |
def to_unixnano(timestamp): |
| 13 |
return int(timestamp) * (10**9) |
| 14 |
|
| 15 |
# include/atheme/channels.h |
| 16 |
CMODE_FLAG_TO_MODE = { |
| 17 |
0x001: 'i', # CMODE_INVITE |
| 18 |
0x010: 'n', # CMODE_NOEXT |
| 19 |
0x080: 's', # CMODE_SEC |
| 20 |
0x100: 't', # CMODE_TOPIC |
| 21 |
} |
| 22 |
|
| 23 |
# attempt to interpret certfp as a hex-encoded SHA-256 fingerprint |
| 24 |
def validate_certfp(certfp): |
| 25 |
try: |
| 26 |
dec = binascii.unhexlify(certfp) |
| 27 |
except: |
| 28 |
return False |
| 29 |
return len(dec) == 32 |
| 30 |
|
| 31 |
def convert(infile): |
| 32 |
out = { |
| 33 |
'version': 1, |
| 34 |
'source': 'atheme', |
| 35 |
'users': defaultdict(dict), |
| 36 |
'channels': defaultdict(dict), |
| 37 |
} |
| 38 |
|
| 39 |
group_to_founders = defaultdict(list) |
| 40 |
|
| 41 |
channel_to_founder = defaultdict(lambda: (None, None)) |
| 42 |
|
| 43 |
while True: |
| 44 |
line = infile.readline() |
| 45 |
if not line: |
| 46 |
break |
| 47 |
line = line.rstrip(b'\r\n') |
| 48 |
try: |
| 49 |
line = line.decode('utf-8') |
| 50 |
except UnicodeDecodeError: |
| 51 |
line = line.decode('utf-8', 'replace') |
| 52 |
logging.warning("line contained invalid utf8 data " + line) |
| 53 |
parts = line.split(' ') |
| 54 |
category = parts[0] |
| 55 |
|
| 56 |
if category == 'GACL': |
| 57 |
# Note: all group definitions precede channel access entries (token CA) by design, so it |
| 58 |
# should be safe to read this in using one pass. |
| 59 |
groupname = parts[1] |
| 60 |
user = parts[2] |
| 61 |
flags = parts[3] |
| 62 |
if 'F' in flags: |
| 63 |
group_to_founders[groupname].append(user) |
| 64 |
elif category == 'MU': |
| 65 |
# user account |
| 66 |
# MU AAAAAAAAB shivaram $1$hcspif$nCm4r3S14Me9ifsOPGuJT. [email protected] 1600134392 1600467343 +sC default |
| 67 |
name = parts[2] |
| 68 |
user = {'name': name, 'hash': parts[3], 'email': parts[4], 'registeredAt': to_unixnano(parts[5])} |
| 69 |
out['users'][name].update(user) |
| 70 |
pass |
| 71 |
elif category == 'MN': |
| 72 |
# grouped nick |
| 73 |
# MN shivaram slingamn 1600218831 1600467343 |
| 74 |
username, groupednick = parts[1], parts[2] |
| 75 |
if username != groupednick: |
| 76 |
user = out['users'][username] |
| 77 |
user.setdefault('additionalnicks', []).append(groupednick) |
| 78 |
elif category == 'MDU': |
| 79 |
if parts[2] == 'private:usercloak': |
| 80 |
username = parts[1] |
| 81 |
out['users'][username]['vhost'] = parts[3] |
| 82 |
elif category == 'MCFP': |
| 83 |
username, certfp = parts[1], parts[2] |
| 84 |
if validate_certfp(certfp): |
| 85 |
user = out['users'][username] |
| 86 |
user.setdefault('certfps', []).append(certfp.lower()) |
| 87 |
elif category == 'MC': |
| 88 |
# channel registration |
| 89 |
# MC #mychannel 1600134478 1600467343 +v 272 0 0 |
| 90 |
# MC #NEWCHANNELTEST 1602270889 1602270974 +vg 1 0 0 jaeger4 |
| 91 |
chname = parts[1] |
| 92 |
chdata = out['channels'][chname] |
| 93 |
# XXX just give everyone +nt, regardless of lock status; they can fix it later |
| 94 |
chdata.update({'name': chname, 'registeredAt': to_unixnano(parts[2])}) |
| 95 |
if parts[8] != '': |
| 96 |
chdata['key'] = parts[8] |
| 97 |
modes = {'n', 't'} |
| 98 |
mlock_on, mlock_off = int(parts[5]), int(parts[6]) |
| 99 |
for flag, mode in CMODE_FLAG_TO_MODE.items(): |
| 100 |
if flag & mlock_on != 0: |
| 101 |
modes.add(mode) |
| 102 |
elif flag & mlock_off != 0 and mode in modes: |
| 103 |
modes.remove(mode) |
| 104 |
chdata['modes'] = ''.join(sorted(modes)) |
| 105 |
chdata['limit'] = int(parts[7]) |
| 106 |
elif category == 'MDC': |
| 107 |
# auxiliary data for a channel registration |
| 108 |
# MDC #mychannel private:topic:setter s |
| 109 |
# MDC #mychannel private:topic:text hi again |
| 110 |
# MDC #mychannel private:topic:ts 1600135864 |
| 111 |
chname = parts[1] |
| 112 |
category = parts[2] |
| 113 |
if category == 'private:topic:text': |
| 114 |
out['channels'][chname]['topic'] = line.split(maxsplit=3)[3] |
| 115 |
elif category == 'private:topic:setter': |
| 116 |
out['channels'][chname]['topicSetBy'] = parts[3] |
| 117 |
elif category == 'private:topic:ts': |
| 118 |
out['channels'][chname]['topicSetAt'] = to_unixnano(parts[3]) |
| 119 |
elif category == 'private:mlockext': |
| 120 |
# the channel forward mode is +L on insp/unreal, +f on charybdis |
| 121 |
# charybdis has a +L ("large banlist") taking no argument |
| 122 |
# and unreal has a +f ("flood limit") taking two colon-delimited numbers, |
| 123 |
# so check for an argument that starts with a # |
| 124 |
if parts[3].startswith('L#') or parts[3].startswith('f#'): |
| 125 |
out['channels'][chname]['forward'] = parts[3][1:] |
| 126 |
elif category == 'CA': |
| 127 |
# channel access lists |
| 128 |
# CA #mychannel shivaram +AFORafhioqrstv 1600134478 shivaram |
| 129 |
chname, username, flags, set_at = parts[1], parts[2], parts[3], int(parts[4]) |
| 130 |
chname = parts[1] |
| 131 |
chdata = out['channels'][chname] |
| 132 |
flags = parts[3] |
| 133 |
set_at = int(parts[4]) |
| 134 |
if 'amode' not in chdata: |
| 135 |
chdata['amode'] = {} |
| 136 |
# see libathemecore/flags.c: +o is op, +O is autoop, etc. |
| 137 |
if 'F' in flags: |
| 138 |
# If the username starts with "!", it's actually a GroupServ group. |
| 139 |
if username.startswith('!'): |
| 140 |
group_founders = group_to_founders.get(username) |
| 141 |
if not group_founders: |
| 142 |
# skip this and warn about it later |
| 143 |
continue |
| 144 |
# attempt to promote the first group founder to channel founder |
| 145 |
username = group_founders[0] |
| 146 |
# but everyone gets the +q flag |
| 147 |
for founder in group_founders: |
| 148 |
chdata['amode'][founder] = 'q' |
| 149 |
# there can only be one founder |
| 150 |
preexisting_founder, preexisting_set_at = channel_to_founder[chname] |
| 151 |
if preexisting_founder is None or set_at < preexisting_set_at: |
| 152 |
chdata['founder'] = username |
| 153 |
channel_to_founder[chname] = (username, set_at) |
| 154 |
# but multiple people can receive the 'q' amode |
| 155 |
chdata['amode'][username] = 'q' |
| 156 |
continue |
| 157 |
if MASK_MAGIC_REGEX.search(username): |
| 158 |
# ignore groups, masks, etc. for any field other than founder |
| 159 |
continue |
| 160 |
# record the first appearing successor, if necessary |
| 161 |
if 'S' in flags: |
| 162 |
if not chdata.get('successor'): |
| 163 |
chdata['successor'] = username |
| 164 |
# finally, handle amodes |
| 165 |
if 'q' in flags: |
| 166 |
chdata['amode'][username] = 'q' |
| 167 |
elif 'a' in flags: |
| 168 |
chdata['amode'][username] = 'a' |
| 169 |
elif 'o' in flags or 'O' in flags: |
| 170 |
chdata['amode'][username] = 'o' |
| 171 |
elif 'h' in flags or 'H' in flags: |
| 172 |
chdata['amode'][username] = 'h' |
| 173 |
elif 'v' in flags or 'V' in flags: |
| 174 |
chdata['amode'][username] = 'v' |
| 175 |
else: |
| 176 |
pass |
| 177 |
|
| 178 |
# do some basic integrity checks |
| 179 |
def validate_user(name): |
| 180 |
if not name: |
| 181 |
return False |
| 182 |
return bool(out['users'].get(name)) |
| 183 |
|
| 184 |
invalid_channels = [] |
| 185 |
|
| 186 |
for chname, chdata in out['channels'].items(): |
| 187 |
if not validate_user(chdata.get('founder')): |
| 188 |
if validate_user(chdata.get('successor')): |
| 189 |
chdata['founder'] = chdata['successor'] |
| 190 |
else: |
| 191 |
invalid_channels.append(chname) |
| 192 |
|
| 193 |
for chname in invalid_channels: |
| 194 |
logging.warning("Unable to find a valid founder for channel %s, discarding it", chname) |
| 195 |
del out['channels'][chname] |
| 196 |
|
| 197 |
return out |
| 198 |
|
| 199 |
def main(): |
| 200 |
if len(sys.argv) != 3: |
| 201 |
raise Exception("Usage: atheme2json.py atheme_db output.json") |
| 202 |
with open(sys.argv[1], 'rb') as infile: |
| 203 |
output = convert(infile) |
| 204 |
with open(sys.argv[2], 'w') as outfile: |
| 205 |
json.dump(output, outfile) |
| 206 |
|
| 207 |
if __name__ == '__main__': |
| 208 |
logging.basicConfig() |
| 209 |
sys.exit(main()) |