Debian Codegraph @

morfologik-tools/src/main/java/morfologik/tools/DictCompile.java

raw
1 package morfologik.tools;
2
3 import java.io.BufferedInputStream;
4 import java.io.BufferedOutputStream;
5 import java.io.InputStream;
6 import java.io.OutputStream;
7 import java.nio.ByteBuffer;
8 import java.nio.charset.CharsetDecoder;
9 import java.nio.charset.CodingErrorAction;
10 import java.nio.file.Files;
11 import java.nio.file.Path;
12 import java.util.Collections;
13 import java.util.Iterator;
14 import java.util.List;
15
16 import com.beust.jcommander.Parameter;
17 import com.beust.jcommander.Parameters;
18 import com.beust.jcommander.ParametersDelegate;
19
20 import morfologik.fsa.FSA;
21 import morfologik.fsa.builders.FSABuilder;
22 import morfologik.fsa.builders.FSASerializer;
23 import morfologik.stemming.BufferUtils;
24 import morfologik.stemming.Dictionary;
25 import morfologik.stemming.DictionaryLookup;
26 import morfologik.stemming.DictionaryMetadata;
27 import morfologik.stemming.ISequenceEncoder;
28
29 /**
30 * Decompiles morphological dictionary automaton back to source state.
31 */
32 @Parameters(
33 commandNames = "dict_compile",
34 commandDescription = "Compiles a morphological dictionary automaton.")
35 public class DictCompile extends CliTool {
36 @Parameter(
37 names = {"-i", "--input"},
38 description = "The input file (base,inflected,tag). An associated metadata (*.info) file must exist.",
39 required = true,
40 validateValueWith = ValidateFileExists.class)
41 private Path input;
42
43 @Parameter(
44 names = ARG_VALIDATE,
45 arity = 1,
46 description = "Validate input to make sure it makes sense.")
47 private boolean validate = true;
48
49 @Parameter(
50 names = {"-f", "--format"},
51 description = "Automaton serialization format.")
52 private SerializationFormat format = SerializationFormat.FSA5;
53
54 @Parameter(
55 names = ARG_OVERWRITE,
56 description = "Overwrite the output file if it exists.")
57 private boolean overwrite;
58
59 @ParametersDelegate
60 private final BinaryInput binaryInput;
61
62 DictCompile() {
63 binaryInput = new BinaryInput();
64 }
65
66 public DictCompile(Path input,
67 boolean overwrite,
68 boolean validate,
69 boolean acceptBom,
70 boolean acceptCr,
71 boolean ignoreEmpty) {
72 this.input = checkNotNull(input);
73 this.overwrite = overwrite;
74 this.validate = validate;
75 this.binaryInput = new BinaryInput(acceptBom, acceptCr, ignoreEmpty);
76 }
77
78 @Override
79 public ExitStatus call() throws Exception {
80 final Path metadataPath = DictionaryMetadata.getExpectedMetadataLocation(input);
81
82 if (!Files.isRegularFile(metadataPath)) {
83 System.err.println("Dictionary metadata file for the input does not exist: " + metadataPath);
84 System.err.println("The metadata file (with at least the column separator and byte encoding) "
85 + "is required. Check out the examples.");
86 return ExitStatus.ERROR_OTHER;
87 }
88
89 final Path output = metadataPath.resolveSibling(
90 metadataPath.getFileName().toString().replaceAll(
91 "\\." + DictionaryMetadata.METADATA_FILE_EXTENSION + "$", ".dict"));
92
93 if (!overwrite && Files.exists(output)) {
94 throw new ExitStatusException(ExitStatus.ERROR_CONFIRMATION_REQUIRED,
95 "Output dictionary file already exists: %s, use %s to override.", output, ARG_OVERWRITE);
96 }
97
98 final DictionaryMetadata metadata;
99 try (InputStream is = new BufferedInputStream(Files.newInputStream(metadataPath))) {
100 metadata = DictionaryMetadata.read(is);
101 }
102
103 final List<byte[]> sequences = binaryInput.readBinarySequences(input, (byte) '\n');
104
105 final CharsetDecoder charsetDecoder = metadata.getDecoder()
106 .onMalformedInput(CodingErrorAction.REPORT)
107 .onUnmappableCharacter(CodingErrorAction.REPORT);
108
109 final byte separator = metadata.getSeparator();
110 final ISequenceEncoder sequenceEncoder = metadata.getSequenceEncoderType().get();
111
112 if (!sequences.isEmpty()) {
113 Iterator<byte[]> i = sequences.iterator();
114 byte [] row = i.next();
115 final int separatorCount = countOf(separator, row);
116
117 if (separatorCount < 1 || separatorCount > 2) {
118 throw new ExitStatusException(ExitStatus.ERROR_OTHER,
119 "Invalid input. Each row must consist of [base,inflected,tag?] columns, where ',' is a "
120 + "separator character (declared as: %s). This row contains %d separator characters: %s",
121 Character.isJavaIdentifierPart(metadata.getSeparatorAsChar())
122 ? "'" + Character.toString(metadata.getSeparatorAsChar()) + "'"
123 : "0x" + Integer.toHexString((int) separator & 0xff),
124 separatorCount,
125 new String(row, charsetDecoder.charset()));
126 }
127
128 while (i.hasNext()) {
129 row = i.next();
130 int count = countOf(separator, row);
131 if (count != separatorCount) {
132 throw new ExitStatusException(ExitStatus.ERROR_OTHER,
133 "The number of separators (%d) is inconsistent with previous lines: %s",
134 count,
135 new String(row, charsetDecoder.charset()));
136 }
137 }
138 }
139
140 ByteBuffer encoded = ByteBuffer.allocate(0);
141 ByteBuffer source = ByteBuffer.allocate(0);
142 ByteBuffer target = ByteBuffer.allocate(0);
143 ByteBuffer tag = ByteBuffer.allocate(0);
144 ByteBuffer assembled = ByteBuffer.allocate(0);
145 for (int i = 0, max = sequences.size(); i < max; i++) {
146 byte[] row = sequences.get(i);
147 int sep1 = indexOf(separator, row, 0);
148 int sep2 = indexOf(separator, row, sep1 + 1);
149 if (sep2 < 0) {
150 sep2 = row.length;
151 }
152
153 source = BufferUtils.clearAndEnsureCapacity(source, sep1);
154 source.put(row, 0, sep1);
155 source.flip();
156
157 final int len = sep2 - (sep1 + 1);
158 target = BufferUtils.clearAndEnsureCapacity(target, len);
159 target.put(row, sep1 + 1, len);
160 target.flip();
161
162 final int len2 = row.length - (sep2 + 1);
163 tag = BufferUtils.clearAndEnsureCapacity(tag, len2);
164 if (len2 > 0) {
165 tag.put(row, sep2 + 1, len2);
166 }
167 tag.flip();
168
169 encoded = sequenceEncoder.encode(encoded, target, source);
170
171 assembled = BufferUtils.clearAndEnsureCapacity(assembled,
172 target.remaining() + 1 +
173 encoded.remaining() + 1 +
174 tag.remaining());
175
176 assembled.put(target);
177 assembled.put(separator);
178 assembled.put(encoded);
179 if (tag.hasRemaining()) {
180 assembled.put(separator);
181 assembled.put(tag);
182 }
183 assembled.flip();
184
185 sequences.set(i, BufferUtils.toArray(assembled));
186 }
187
188 Collections.sort(sequences, FSABuilder.LEXICAL_ORDERING);
189 FSA fsa = FSABuilder.build(sequences);
190
191 FSASerializer serializer = format.getSerializer();
192 try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(output))) {
193 serializer.serialize(fsa, os);
194 }
195
196 // If validating, try to scan the input
197 if (validate) {
198 DictionaryLookup dictionaryLookup = new DictionaryLookup(new Dictionary(fsa, metadata));
199 for (Iterator<?> i = dictionaryLookup.iterator(); i.hasNext(); i.next()) {
200 // Do nothing, just scan and make sure no exceptions are thrown.
201 }
202 }
203
204 return ExitStatus.SUCCESS;
205 }
206
207 private static int countOf(byte separator, byte[] row) {
208 int cnt = 0;
209 for (int i = row.length; --i >= 0;) {
210 if (row[i] == separator) {
211 cnt++;
212 }
213 }
214 return cnt;
215 }
216
217 private static int indexOf(byte separator, byte[] row, int fromIndex) {
218 while (fromIndex < row.length) {
219 if (row[fromIndex] == separator) {
220 return fromIndex;
221 }
222 fromIndex++;
223 }
224 return -1;
225 }
226
227 public static void main(String[] args) {
228 main(args, new DictCompile());
229 }
230 }