| 1 |
package morfologik.tools; |
| 2 |
|
| 3 |
import java.io.BufferedInputStream; |
| 4 |
import java.io.BufferedOutputStream; |
| 5 |
import java.io.InputStream; |
| 6 |
import java.io.OutputStream; |
| 7 |
import java.nio.ByteBuffer; |
| 8 |
import java.nio.charset.CharsetDecoder; |
| 9 |
import java.nio.charset.CodingErrorAction; |
| 10 |
import java.nio.file.Files; |
| 11 |
import java.nio.file.Path; |
| 12 |
import java.util.Collections; |
| 13 |
import java.util.Iterator; |
| 14 |
import java.util.List; |
| 15 |
|
| 16 |
import com.beust.jcommander.Parameter; |
| 17 |
import com.beust.jcommander.Parameters; |
| 18 |
import com.beust.jcommander.ParametersDelegate; |
| 19 |
|
| 20 |
import morfologik.fsa.FSA; |
| 21 |
import morfologik.fsa.builders.FSABuilder; |
| 22 |
import morfologik.fsa.builders.FSASerializer; |
| 23 |
import morfologik.stemming.BufferUtils; |
| 24 |
import morfologik.stemming.Dictionary; |
| 25 |
import morfologik.stemming.DictionaryLookup; |
| 26 |
import morfologik.stemming.DictionaryMetadata; |
| 27 |
import morfologik.stemming.ISequenceEncoder; |
| 28 |
|
| 29 |
/** |
| 30 |
* Decompiles morphological dictionary automaton back to source state. |
| 31 |
*/ |
| 32 |
@Parameters( |
| 33 |
commandNames = "dict_compile", |
| 34 |
commandDescription = "Compiles a morphological dictionary automaton.") |
| 35 |
public class DictCompile extends CliTool { |
| 36 |
@Parameter( |
| 37 |
names = {"-i", "--input"}, |
| 38 |
description = "The input file (base,inflected,tag). An associated metadata (*.info) file must exist.", |
| 39 |
required = true, |
| 40 |
validateValueWith = ValidateFileExists.class) |
| 41 |
private Path input; |
| 42 |
|
| 43 |
@Parameter( |
| 44 |
names = ARG_VALIDATE, |
| 45 |
arity = 1, |
| 46 |
description = "Validate input to make sure it makes sense.") |
| 47 |
private boolean validate = true; |
| 48 |
|
| 49 |
@Parameter( |
| 50 |
names = {"-f", "--format"}, |
| 51 |
description = "Automaton serialization format.") |
| 52 |
private SerializationFormat format = SerializationFormat.FSA5; |
| 53 |
|
| 54 |
@Parameter( |
| 55 |
names = ARG_OVERWRITE, |
| 56 |
description = "Overwrite the output file if it exists.") |
| 57 |
private boolean overwrite; |
| 58 |
|
| 59 |
@ParametersDelegate |
| 60 |
private final BinaryInput binaryInput; |
| 61 |
|
| 62 |
DictCompile() { |
| 63 |
binaryInput = new BinaryInput(); |
| 64 |
} |
| 65 |
|
| 66 |
public DictCompile(Path input, |
| 67 |
boolean overwrite, |
| 68 |
boolean validate, |
| 69 |
boolean acceptBom, |
| 70 |
boolean acceptCr, |
| 71 |
boolean ignoreEmpty) { |
| 72 |
this.input = checkNotNull(input); |
| 73 |
this.overwrite = overwrite; |
| 74 |
this.validate = validate; |
| 75 |
this.binaryInput = new BinaryInput(acceptBom, acceptCr, ignoreEmpty); |
| 76 |
} |
| 77 |
|
| 78 |
@Override |
| 79 |
public ExitStatus call() throws Exception { |
| 80 |
final Path metadataPath = DictionaryMetadata.getExpectedMetadataLocation(input); |
| 81 |
|
| 82 |
if (!Files.isRegularFile(metadataPath)) { |
| 83 |
System.err.println("Dictionary metadata file for the input does not exist: " + metadataPath); |
| 84 |
System.err.println("The metadata file (with at least the column separator and byte encoding) " |
| 85 |
+ "is required. Check out the examples."); |
| 86 |
return ExitStatus.ERROR_OTHER; |
| 87 |
} |
| 88 |
|
| 89 |
final Path output = metadataPath.resolveSibling( |
| 90 |
metadataPath.getFileName().toString().replaceAll( |
| 91 |
"\\." + DictionaryMetadata.METADATA_FILE_EXTENSION + "$", ".dict")); |
| 92 |
|
| 93 |
if (!overwrite && Files.exists(output)) { |
| 94 |
throw new ExitStatusException(ExitStatus.ERROR_CONFIRMATION_REQUIRED, |
| 95 |
"Output dictionary file already exists: %s, use %s to override.", output, ARG_OVERWRITE); |
| 96 |
} |
| 97 |
|
| 98 |
final DictionaryMetadata metadata; |
| 99 |
try (InputStream is = new BufferedInputStream(Files.newInputStream(metadataPath))) { |
| 100 |
metadata = DictionaryMetadata.read(is); |
| 101 |
} |
| 102 |
|
| 103 |
final List<byte[]> sequences = binaryInput.readBinarySequences(input, (byte) '\n'); |
| 104 |
|
| 105 |
final CharsetDecoder charsetDecoder = metadata.getDecoder() |
| 106 |
.onMalformedInput(CodingErrorAction.REPORT) |
| 107 |
.onUnmappableCharacter(CodingErrorAction.REPORT); |
| 108 |
|
| 109 |
final byte separator = metadata.getSeparator(); |
| 110 |
final ISequenceEncoder sequenceEncoder = metadata.getSequenceEncoderType().get(); |
| 111 |
|
| 112 |
if (!sequences.isEmpty()) { |
| 113 |
Iterator<byte[]> i = sequences.iterator(); |
| 114 |
byte [] row = i.next(); |
| 115 |
final int separatorCount = countOf(separator, row); |
| 116 |
|
| 117 |
if (separatorCount < 1 || separatorCount > 2) { |
| 118 |
throw new ExitStatusException(ExitStatus.ERROR_OTHER, |
| 119 |
"Invalid input. Each row must consist of [base,inflected,tag?] columns, where ',' is a " |
| 120 |
+ "separator character (declared as: %s). This row contains %d separator characters: %s", |
| 121 |
Character.isJavaIdentifierPart(metadata.getSeparatorAsChar()) |
| 122 |
? "'" + Character.toString(metadata.getSeparatorAsChar()) + "'" |
| 123 |
: "0x" + Integer.toHexString((int) separator & 0xff), |
| 124 |
separatorCount, |
| 125 |
new String(row, charsetDecoder.charset())); |
| 126 |
} |
| 127 |
|
| 128 |
while (i.hasNext()) { |
| 129 |
row = i.next(); |
| 130 |
int count = countOf(separator, row); |
| 131 |
if (count != separatorCount) { |
| 132 |
throw new ExitStatusException(ExitStatus.ERROR_OTHER, |
| 133 |
"The number of separators (%d) is inconsistent with previous lines: %s", |
| 134 |
count, |
| 135 |
new String(row, charsetDecoder.charset())); |
| 136 |
} |
| 137 |
} |
| 138 |
} |
| 139 |
|
| 140 |
ByteBuffer encoded = ByteBuffer.allocate(0); |
| 141 |
ByteBuffer source = ByteBuffer.allocate(0); |
| 142 |
ByteBuffer target = ByteBuffer.allocate(0); |
| 143 |
ByteBuffer tag = ByteBuffer.allocate(0); |
| 144 |
ByteBuffer assembled = ByteBuffer.allocate(0); |
| 145 |
for (int i = 0, max = sequences.size(); i < max; i++) { |
| 146 |
byte[] row = sequences.get(i); |
| 147 |
int sep1 = indexOf(separator, row, 0); |
| 148 |
int sep2 = indexOf(separator, row, sep1 + 1); |
| 149 |
if (sep2 < 0) { |
| 150 |
sep2 = row.length; |
| 151 |
} |
| 152 |
|
| 153 |
source = BufferUtils.clearAndEnsureCapacity(source, sep1); |
| 154 |
source.put(row, 0, sep1); |
| 155 |
source.flip(); |
| 156 |
|
| 157 |
final int len = sep2 - (sep1 + 1); |
| 158 |
target = BufferUtils.clearAndEnsureCapacity(target, len); |
| 159 |
target.put(row, sep1 + 1, len); |
| 160 |
target.flip(); |
| 161 |
|
| 162 |
final int len2 = row.length - (sep2 + 1); |
| 163 |
tag = BufferUtils.clearAndEnsureCapacity(tag, len2); |
| 164 |
if (len2 > 0) { |
| 165 |
tag.put(row, sep2 + 1, len2); |
| 166 |
} |
| 167 |
tag.flip(); |
| 168 |
|
| 169 |
encoded = sequenceEncoder.encode(encoded, target, source); |
| 170 |
|
| 171 |
assembled = BufferUtils.clearAndEnsureCapacity(assembled, |
| 172 |
target.remaining() + 1 + |
| 173 |
encoded.remaining() + 1 + |
| 174 |
tag.remaining()); |
| 175 |
|
| 176 |
assembled.put(target); |
| 177 |
assembled.put(separator); |
| 178 |
assembled.put(encoded); |
| 179 |
if (tag.hasRemaining()) { |
| 180 |
assembled.put(separator); |
| 181 |
assembled.put(tag); |
| 182 |
} |
| 183 |
assembled.flip(); |
| 184 |
|
| 185 |
sequences.set(i, BufferUtils.toArray(assembled)); |
| 186 |
} |
| 187 |
|
| 188 |
Collections.sort(sequences, FSABuilder.LEXICAL_ORDERING); |
| 189 |
FSA fsa = FSABuilder.build(sequences); |
| 190 |
|
| 191 |
FSASerializer serializer = format.getSerializer(); |
| 192 |
try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(output))) { |
| 193 |
serializer.serialize(fsa, os); |
| 194 |
} |
| 195 |
|
| 196 |
// If validating, try to scan the input |
| 197 |
if (validate) { |
| 198 |
DictionaryLookup dictionaryLookup = new DictionaryLookup(new Dictionary(fsa, metadata)); |
| 199 |
for (Iterator<?> i = dictionaryLookup.iterator(); i.hasNext(); i.next()) { |
| 200 |
// Do nothing, just scan and make sure no exceptions are thrown. |
| 201 |
} |
| 202 |
} |
| 203 |
|
| 204 |
return ExitStatus.SUCCESS; |
| 205 |
} |
| 206 |
|
| 207 |
private static int countOf(byte separator, byte[] row) { |
| 208 |
int cnt = 0; |
| 209 |
for (int i = row.length; --i >= 0;) { |
| 210 |
if (row[i] == separator) { |
| 211 |
cnt++; |
| 212 |
} |
| 213 |
} |
| 214 |
return cnt; |
| 215 |
} |
| 216 |
|
| 217 |
private static int indexOf(byte separator, byte[] row, int fromIndex) { |
| 218 |
while (fromIndex < row.length) { |
| 219 |
if (row[fromIndex] == separator) { |
| 220 |
return fromIndex; |
| 221 |
} |
| 222 |
fromIndex++; |
| 223 |
} |
| 224 |
return -1; |
| 225 |
} |
| 226 |
|
| 227 |
public static void main(String[] args) { |
| 228 |
main(args, new DictCompile()); |
| 229 |
} |
| 230 |
} |