| 1 |
package morfologik.tools; |
| 2 |
|
| 3 |
import java.io.BufferedInputStream; |
| 4 |
import java.io.BufferedReader; |
| 5 |
import java.io.Closeable; |
| 6 |
import java.io.Console; |
| 7 |
import java.io.IOException; |
| 8 |
import java.io.InputStreamReader; |
| 9 |
import java.nio.charset.Charset; |
| 10 |
import java.nio.file.Files; |
| 11 |
import java.nio.file.Path; |
| 12 |
import java.util.List; |
| 13 |
|
| 14 |
import com.beust.jcommander.Parameter; |
| 15 |
import com.beust.jcommander.Parameters; |
| 16 |
|
| 17 |
import morfologik.stemming.Dictionary; |
| 18 |
import morfologik.stemming.DictionaryLookup; |
| 19 |
import morfologik.stemming.WordData; |
| 20 |
|
| 21 |
/** |
| 22 |
* Applies a morphological dictionary automaton to the input. |
| 23 |
*/ |
| 24 |
@Parameters( |
| 25 |
commandNames = "dict_apply", |
| 26 |
commandDescription = "Applies a dictionary to an input. Each line is considered an input term.") |
| 27 |
public class DictApply extends CliTool { |
| 28 |
private final static String ARG_ENCODING = "--input-charset"; |
| 29 |
|
| 30 |
@Parameter( |
| 31 |
names = {"-i", "--input"}, |
| 32 |
required = false, |
| 33 |
description = "The input file, each entry in a single line. If not provided, stdin is used.", |
| 34 |
validateValueWith = ValidateFileExists.class) |
| 35 |
private Path input; |
| 36 |
|
| 37 |
@Parameter( |
| 38 |
names = {"-d", "--dictionary"}, |
| 39 |
description = "The dictionary (*.dict and a sibling *.info metadata) to apply.", |
| 40 |
required = true, |
| 41 |
validateValueWith = ValidateFileExists.class) |
| 42 |
private Path dictionary; |
| 43 |
|
| 44 |
@Parameter( |
| 45 |
names = {ARG_ENCODING}, |
| 46 |
required = false, |
| 47 |
description = "Character encoding of the input (platform's default).") |
| 48 |
private String inputEncoding; |
| 49 |
|
| 50 |
@Parameter( |
| 51 |
names = {"--skip-tags"}, |
| 52 |
required = false, |
| 53 |
description = "Skip tags in the output, only print base forms if found.") |
| 54 |
private boolean skipTags = false; |
| 55 |
|
| 56 |
private abstract class LineSupplier implements Closeable { |
| 57 |
public abstract String nextLine() throws IOException; |
| 58 |
|
| 59 |
@Override |
| 60 |
public void close() throws IOException { |
| 61 |
// No-op by default. |
| 62 |
} |
| 63 |
} |
| 64 |
|
| 65 |
private class ReaderLineSupplier extends LineSupplier { |
| 66 |
private final BufferedReader lineReader; |
| 67 |
|
| 68 |
public ReaderLineSupplier(BufferedReader reader) { |
| 69 |
this.lineReader = reader; |
| 70 |
} |
| 71 |
|
| 72 |
@Override |
| 73 |
public String nextLine() throws IOException { |
| 74 |
return lineReader.readLine(); |
| 75 |
} |
| 76 |
|
| 77 |
@Override |
| 78 |
public void close() throws IOException { |
| 79 |
lineReader.close(); |
| 80 |
} |
| 81 |
} |
| 82 |
|
| 83 |
DictApply() { |
| 84 |
} |
| 85 |
|
| 86 |
public DictApply(Path dictionary, |
| 87 |
Path input, |
| 88 |
String inputEncoding) { |
| 89 |
this.input = checkNotNull(input); |
| 90 |
this.dictionary = checkNotNull(dictionary); |
| 91 |
} |
| 92 |
|
| 93 |
@Override |
| 94 |
public ExitStatus call() throws Exception { |
| 95 |
ExitStatus exitStatus = validateArguments(); |
| 96 |
if (exitStatus != null) { |
| 97 |
return exitStatus; |
| 98 |
} |
| 99 |
|
| 100 |
final DictionaryLookup lookup = new DictionaryLookup(Dictionary.read(this.dictionary)); |
| 101 |
try (final LineSupplier input = determineInput()) { |
| 102 |
String line; |
| 103 |
while ((line = input.nextLine()) != null) { |
| 104 |
if (line.length() == 0) { |
| 105 |
continue; |
| 106 |
} |
| 107 |
|
| 108 |
List<WordData> wordData = lookup.lookup(line); |
| 109 |
if (wordData.isEmpty()) { |
| 110 |
System.out.println(line + " => [not found]"); |
| 111 |
} else { |
| 112 |
for (WordData wd : wordData) { |
| 113 |
CharSequence stem = wd.getStem(); |
| 114 |
CharSequence tag = wd.getTag(); |
| 115 |
System.out.println(line + " => " + |
| 116 |
((skipTags || tag == null) ? stem |
| 117 |
: stem + " " + tag)); |
| 118 |
} |
| 119 |
} |
| 120 |
} |
| 121 |
} |
| 122 |
|
| 123 |
return ExitStatus.SUCCESS; |
| 124 |
} |
| 125 |
|
| 126 |
private LineSupplier determineInput() throws IOException { |
| 127 |
if (this.input != null) { |
| 128 |
return new ReaderLineSupplier(Files.newBufferedReader(this.input, Charset.forName(inputEncoding))); |
| 129 |
} |
| 130 |
|
| 131 |
final Console c = System.console(); |
| 132 |
if (c != null) { |
| 133 |
System.err.println("NOTE: Using Console for input, character encoding is unknown but should be all right."); |
| 134 |
return new LineSupplier() { |
| 135 |
@Override |
| 136 |
public String nextLine() throws IOException { |
| 137 |
return c.readLine(); |
| 138 |
} |
| 139 |
}; |
| 140 |
} |
| 141 |
|
| 142 |
Charset charset = this.inputEncoding != null ? Charset.forName(this.inputEncoding) |
| 143 |
: Charset.defaultCharset(); |
| 144 |
System.err.println("NOTE: Using stdin for input, character encoding set to: " + charset.name() + |
| 145 |
" (use " + ARG_ENCODING + " to override)."); |
| 146 |
return new ReaderLineSupplier( |
| 147 |
new BufferedReader( |
| 148 |
new InputStreamReader( |
| 149 |
new BufferedInputStream(System.in), charset))); |
| 150 |
} |
| 151 |
|
| 152 |
private ExitStatus validateArguments() { |
| 153 |
if (this.input != null) { |
| 154 |
if (this.inputEncoding == null) { |
| 155 |
System.err.println("Input encoding is required if file input is used."); |
| 156 |
return ExitStatus.ERROR_INVALID_ARGUMENTS; |
| 157 |
} |
| 158 |
} else { |
| 159 |
if (System.console() != null && this.inputEncoding != null) { |
| 160 |
System.err.println("Input encoding is only valid with file input or stdin redirection."); |
| 161 |
return ExitStatus.ERROR_INVALID_ARGUMENTS; |
| 162 |
} |
| 163 |
} |
| 164 |
|
| 165 |
return null; |
| 166 |
} |
| 167 |
|
| 168 |
public static void main(String[] args) { |
| 169 |
main(args, new DictApply()); |
| 170 |
} |
| 171 |
} |