Debian Codegraph @

tools/import_corpus.py

raw
1 #!/usr/bin/python3
2
3 import argparse
4 import base64
5 import hashlib
6 import os
7 from collections import defaultdict
8
9 def hash_line(s):
10 return base64.b64encode(hashlib.sha256(s.encode('utf-8')).digest(), b'-_')[:12].decode('utf-8')
11
12 parser = argparse.ArgumentParser('convert a text corpus to apertium-regtest inputs and golds')
13 parser.add_argument('test', help='test corpus to create or append to')
14 parser.add_argument('corp', help='input corpus to convert')
15 parser.add_argument('-d', '--dir', help='location of test directory (default: current directory)', default='.')
16 parser.add_argument('-s', '--sep', help='separator between input and gold (default: newline)', default='\n')
17 args = parser.parse_args()
18
19 inp = []
20 gold = defaultdict(list)
21
22 with open(args.corp) as fin:
23 line_sep = '\n\n' if args.sep == '\n' else '\n'
24 txt = fin.read().split(line_sep)
25 for line in txt:
26 ls = line.strip().split(args.sep)
27 if len(ls) < 2:
28 continue
29 i = ls[0]
30 g = ls[1]
31 inp.append(i)
32 gold[hash_line(i)].append(g)
33
34 pth = os.path.join(args.dir, 'test')
35 if not os.path.isdir(pth):
36 os.mkdir(pth)
37 os.chdir(pth)
38
39 with open(args.test + '-input.txt', 'a') as fout:
40 fout.write('\n' + '\n'.join(inp))
41 print('Inputs written to', os.path.join(pth, args.test + '-input.txt'))
42
43 with open(args.test + '-generator-gold.txt', 'a') as fout:
44 fout.write('\n')
45 for k in sorted(gold.keys()):
46 fout.write('[%s]\n' % k)
47 fout.write(''.join(g + ' [/option]\n' for g in gold[k]))
48 fout.write('[/%s]\n' % k)
49 print('Gold values written to', os.path.join(pth, args.test + '-generator-gold.txt'))
50
51 print('If either of these is the wrong file, the contents can be safely copied to the end of the correct file(s).')