Debian Codegraph @

morfologik-tools/src/main/java/morfologik/tools/DictDecompile.java

raw
1 package morfologik.tools;
2
3 import java.io.BufferedOutputStream;
4 import java.io.IOException;
5 import java.io.OutputStream;
6 import java.nio.ByteBuffer;
7 import java.nio.file.Files;
8 import java.nio.file.Path;
9
10 import morfologik.stemming.Dictionary;
11 import morfologik.stemming.DictionaryLookup;
12 import morfologik.stemming.WordData;
13
14 import com.beust.jcommander.Parameter;
15 import com.beust.jcommander.Parameters;
16
17 /**
18 * Decompiles morphological dictionary automaton back to source state.
19 */
20 @Parameters(
21 commandNames = "dict_decompile",
22 commandDescription = "Decompiles morphological dictionary automaton back to source state.")
23 public class DictDecompile extends CliTool {
24 @Parameter(
25 names = {"-i", "--input"},
26 description = "The input dictionary (*.dict and a sibling *.info metadata).",
27 required = true,
28 validateValueWith = ValidateFileExists.class)
29 private Path input;
30
31 @Parameter(
32 names = {"-o", "--output"},
33 description = "The output file for dictionary data.")
34 private Path output;
35
36 @Parameter(
37 names = ARG_OVERWRITE,
38 description = "Overwrite the output file if it exists.")
39 private boolean overwrite;
40
41 @Parameter(
42 names = ARG_VALIDATE,
43 arity = 1,
44 description = "Validate decoded output to make sure it can be re-encoded.")
45 private boolean validate = true;
46
47 DictDecompile() {
48 }
49
50 public DictDecompile(Path input,
51 Path output,
52 boolean overwrite,
53 boolean validate) {
54 this.input = checkNotNull(input);
55 this.output = output;
56 this.overwrite = overwrite;
57 this.validate = validate;
58 }
59
60 @Override
61 public ExitStatus call() throws Exception {
62 final Dictionary dictionary = Dictionary.read(input);
63 final DictionaryLookup lookup = new DictionaryLookup(dictionary);
64
65 if (output == null) {
66 output = input.resolveSibling(input.getFileName().toString().replaceAll("\\.dict$", "") + ".input");
67 if (Files.exists(output) && !overwrite) {
68 System.err.println("ERROR: the default output file location already exists. Use --overwrite or remove"
69 + " the file manually: " + output.toString());
70 return ExitStatus.ERROR_CONFIRMATION_REQUIRED;
71 }
72 }
73
74 final byte separator = dictionary.metadata.getSeparator();
75 ByteBuffer stem = ByteBuffer.allocate(0);
76 ByteBuffer word = ByteBuffer.allocate(0);
77 ByteBuffer tag = ByteBuffer.allocate(0);
78 try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(output))) {
79 boolean hasTags = false;
80 for (WordData wd : lookup) {
81 tag = wd.getTagBytes(tag);
82 if (tag.hasRemaining()) {
83 hasTags = true;
84 break;
85 }
86 }
87
88 for (WordData wd : lookup) {
89 stem = wd.getStemBytes(stem);
90 word = wd.getWordBytes(word);
91 tag = wd.getTagBytes(tag);
92
93 write(os, stem);
94 os.write(separator);
95 write(os, word);
96 if (hasTags) {
97 os.write(separator);
98 write(os, tag);
99 }
100 os.write('\n');
101
102 if (validate && (ensureNoSeparator(stem, separator) ||
103 ensureNoSeparator(word, separator))) {
104 System.err.println("ERROR: The stem or word of a dictionary entry contains separator "
105 + " byte " + FSAInfo.byteAsChar(separator) + ", this will prevent proper re-encoding."
106 + " Add '--validate false' to override. Offending entry: "
107 + wd.getStem() + ", "
108 + wd.getWord());
109 return ExitStatus.ERROR_OTHER;
110 }
111 }
112 }
113
114 return ExitStatus.SUCCESS;
115 }
116
117 private void write(OutputStream os, ByteBuffer bb) throws IOException {
118 os.write(bb.array(), bb.arrayOffset() + bb.position(), bb.remaining());
119 }
120
121 private boolean ensureNoSeparator(ByteBuffer bb, byte marker) {
122 byte [] buf = bb.array();
123 for (int o = bb.arrayOffset() + bb.position(), i = bb.remaining(); i > 0; i--) {
124 if (buf[o] == marker) {
125 return true;
126 }
127 }
128 return false;
129 }
130
131 public static void main(String[] args) {
132 main(args, new DictDecompile());
133 }
134 }