| 1 |
#!/usr/bin/python3 |
| 2 |
|
| 3 |
import argparse |
| 4 |
import base64 |
| 5 |
import hashlib |
| 6 |
import os |
| 7 |
from collections import defaultdict |
| 8 |
|
| 9 |
def hash_line(s): |
| 10 |
return base64.b64encode(hashlib.sha256(s.encode('utf-8')).digest(), b'-_')[:12].decode('utf-8') |
| 11 |
|
| 12 |
parser = argparse.ArgumentParser('convert a text corpus to apertium-regtest inputs and golds') |
| 13 |
parser.add_argument('test', help='test corpus to create or append to') |
| 14 |
parser.add_argument('corp', help='input corpus to convert') |
| 15 |
parser.add_argument('-d', '--dir', help='location of test directory (default: current directory)', default='.') |
| 16 |
parser.add_argument('-s', '--sep', help='separator between input and gold (default: newline)', default='\n') |
| 17 |
args = parser.parse_args() |
| 18 |
|
| 19 |
inp = [] |
| 20 |
gold = defaultdict(list) |
| 21 |
|
| 22 |
with open(args.corp) as fin: |
| 23 |
line_sep = '\n\n' if args.sep == '\n' else '\n' |
| 24 |
txt = fin.read().split(line_sep) |
| 25 |
for line in txt: |
| 26 |
ls = line.strip().split(args.sep) |
| 27 |
if len(ls) < 2: |
| 28 |
continue |
| 29 |
i = ls[0] |
| 30 |
g = ls[1] |
| 31 |
inp.append(i) |
| 32 |
gold[hash_line(i)].append(g) |
| 33 |
|
| 34 |
pth = os.path.join(args.dir, 'test') |
| 35 |
if not os.path.isdir(pth): |
| 36 |
os.mkdir(pth) |
| 37 |
os.chdir(pth) |
| 38 |
|
| 39 |
with open(args.test + '-input.txt', 'a') as fout: |
| 40 |
fout.write('\n' + '\n'.join(inp)) |
| 41 |
print('Inputs written to', os.path.join(pth, args.test + '-input.txt')) |
| 42 |
|
| 43 |
with open(args.test + '-generator-gold.txt', 'a') as fout: |
| 44 |
fout.write('\n') |
| 45 |
for k in sorted(gold.keys()): |
| 46 |
fout.write('[%s]\n' % k) |
| 47 |
fout.write(''.join(g + ' [/option]\n' for g in gold[k])) |
| 48 |
fout.write('[/%s]\n' % k) |
| 49 |
print('Gold values written to', os.path.join(pth, args.test + '-generator-gold.txt')) |
| 50 |
|
| 51 |
print('If either of these is the wrong file, the contents can be safely copied to the end of the correct file(s).') |