| 1 |
package morfologik.tools; |
| 2 |
|
| 3 |
import java.io.BufferedOutputStream; |
| 4 |
import java.io.IOException; |
| 5 |
import java.io.OutputStream; |
| 6 |
import java.nio.ByteBuffer; |
| 7 |
import java.nio.file.Files; |
| 8 |
import java.nio.file.Path; |
| 9 |
|
| 10 |
import morfologik.stemming.Dictionary; |
| 11 |
import morfologik.stemming.DictionaryLookup; |
| 12 |
import morfologik.stemming.WordData; |
| 13 |
|
| 14 |
import com.beust.jcommander.Parameter; |
| 15 |
import com.beust.jcommander.Parameters; |
| 16 |
|
| 17 |
/** |
| 18 |
* Decompiles morphological dictionary automaton back to source state. |
| 19 |
*/ |
| 20 |
@Parameters( |
| 21 |
commandNames = "dict_decompile", |
| 22 |
commandDescription = "Decompiles morphological dictionary automaton back to source state.") |
| 23 |
public class DictDecompile extends CliTool { |
| 24 |
@Parameter( |
| 25 |
names = {"-i", "--input"}, |
| 26 |
description = "The input dictionary (*.dict and a sibling *.info metadata).", |
| 27 |
required = true, |
| 28 |
validateValueWith = ValidateFileExists.class) |
| 29 |
private Path input; |
| 30 |
|
| 31 |
@Parameter( |
| 32 |
names = {"-o", "--output"}, |
| 33 |
description = "The output file for dictionary data.") |
| 34 |
private Path output; |
| 35 |
|
| 36 |
@Parameter( |
| 37 |
names = ARG_OVERWRITE, |
| 38 |
description = "Overwrite the output file if it exists.") |
| 39 |
private boolean overwrite; |
| 40 |
|
| 41 |
@Parameter( |
| 42 |
names = ARG_VALIDATE, |
| 43 |
arity = 1, |
| 44 |
description = "Validate decoded output to make sure it can be re-encoded.") |
| 45 |
private boolean validate = true; |
| 46 |
|
| 47 |
DictDecompile() { |
| 48 |
} |
| 49 |
|
| 50 |
public DictDecompile(Path input, |
| 51 |
Path output, |
| 52 |
boolean overwrite, |
| 53 |
boolean validate) { |
| 54 |
this.input = checkNotNull(input); |
| 55 |
this.output = output; |
| 56 |
this.overwrite = overwrite; |
| 57 |
this.validate = validate; |
| 58 |
} |
| 59 |
|
| 60 |
@Override |
| 61 |
public ExitStatus call() throws Exception { |
| 62 |
final Dictionary dictionary = Dictionary.read(input); |
| 63 |
final DictionaryLookup lookup = new DictionaryLookup(dictionary); |
| 64 |
|
| 65 |
if (output == null) { |
| 66 |
output = input.resolveSibling(input.getFileName().toString().replaceAll("\\.dict$", "") + ".input"); |
| 67 |
if (Files.exists(output) && !overwrite) { |
| 68 |
System.err.println("ERROR: the default output file location already exists. Use --overwrite or remove" |
| 69 |
+ " the file manually: " + output.toString()); |
| 70 |
return ExitStatus.ERROR_CONFIRMATION_REQUIRED; |
| 71 |
} |
| 72 |
} |
| 73 |
|
| 74 |
final byte separator = dictionary.metadata.getSeparator(); |
| 75 |
ByteBuffer stem = ByteBuffer.allocate(0); |
| 76 |
ByteBuffer word = ByteBuffer.allocate(0); |
| 77 |
ByteBuffer tag = ByteBuffer.allocate(0); |
| 78 |
try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(output))) { |
| 79 |
boolean hasTags = false; |
| 80 |
for (WordData wd : lookup) { |
| 81 |
tag = wd.getTagBytes(tag); |
| 82 |
if (tag.hasRemaining()) { |
| 83 |
hasTags = true; |
| 84 |
break; |
| 85 |
} |
| 86 |
} |
| 87 |
|
| 88 |
for (WordData wd : lookup) { |
| 89 |
stem = wd.getStemBytes(stem); |
| 90 |
word = wd.getWordBytes(word); |
| 91 |
tag = wd.getTagBytes(tag); |
| 92 |
|
| 93 |
write(os, stem); |
| 94 |
os.write(separator); |
| 95 |
write(os, word); |
| 96 |
if (hasTags) { |
| 97 |
os.write(separator); |
| 98 |
write(os, tag); |
| 99 |
} |
| 100 |
os.write('\n'); |
| 101 |
|
| 102 |
if (validate && (ensureNoSeparator(stem, separator) || |
| 103 |
ensureNoSeparator(word, separator))) { |
| 104 |
System.err.println("ERROR: The stem or word of a dictionary entry contains separator " |
| 105 |
+ " byte " + FSAInfo.byteAsChar(separator) + ", this will prevent proper re-encoding." |
| 106 |
+ " Add '--validate false' to override. Offending entry: " |
| 107 |
+ wd.getStem() + ", " |
| 108 |
+ wd.getWord()); |
| 109 |
return ExitStatus.ERROR_OTHER; |
| 110 |
} |
| 111 |
} |
| 112 |
} |
| 113 |
|
| 114 |
return ExitStatus.SUCCESS; |
| 115 |
} |
| 116 |
|
| 117 |
private void write(OutputStream os, ByteBuffer bb) throws IOException { |
| 118 |
os.write(bb.array(), bb.arrayOffset() + bb.position(), bb.remaining()); |
| 119 |
} |
| 120 |
|
| 121 |
private boolean ensureNoSeparator(ByteBuffer bb, byte marker) { |
| 122 |
byte [] buf = bb.array(); |
| 123 |
for (int o = bb.arrayOffset() + bb.position(), i = bb.remaining(); i > 0; i--) { |
| 124 |
if (buf[o] == marker) { |
| 125 |
return true; |
| 126 |
} |
| 127 |
} |
| 128 |
return false; |
| 129 |
} |
| 130 |
|
| 131 |
public static void main(String[] args) { |
| 132 |
main(args, new DictDecompile()); |
| 133 |
} |
| 134 |
} |