Debian Codegraph @

morfologik-tools/src/test/java/morfologik/tools/DictCompileTest.java

raw
1 package morfologik.tools;
2
3 import java.io.Writer;
4 import java.nio.charset.StandardCharsets;
5 import java.nio.file.Files;
6 import java.nio.file.Path;
7 import java.util.LinkedHashSet;
8 import java.util.List;
9 import java.util.Set;
10
11 import morfologik.stemming.Dictionary;
12 import morfologik.stemming.DictionaryLookup;
13 import morfologik.stemming.DictionaryMetadata;
14 import morfologik.stemming.EncoderType;
15 import morfologik.stemming.WordData;
16
17 import org.assertj.core.api.Assertions;
18 import org.junit.Test;
19
20 import com.carrotsearch.randomizedtesting.RandomizedTest;
21 import com.carrotsearch.randomizedtesting.annotations.Repeat;
22 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
23
24 public class DictCompileTest extends RandomizedTest {
25 @Test
26 @Repeat(iterations = 200)
27 public void testRoundTrip() throws Exception {
28 final Path input = newTempDir().resolve("dictionary.input");
29 final Path metadata = DictionaryMetadata.getExpectedMetadataLocation(input);
30
31 char separator = RandomPicks.randomFrom(getRandom(), new Character [] {
32 '|',
33 ',',
34 '\t',
35 });
36
37 try (Writer writer = Files.newBufferedWriter(metadata, StandardCharsets.UTF_8)) {
38 DictionaryMetadata.builder()
39 .separator(separator)
40 .encoder(randomFrom(EncoderType.values()))
41 .encoding(StandardCharsets.UTF_8)
42 .build()
43 .write(writer);
44 }
45
46 final boolean useTags = randomBoolean();
47
48 Set<String> sequences = new LinkedHashSet<>();
49 for (int seqs = randomIntBetween(0, 100); --seqs >= 0;) {
50 String base;
51 switch (randomIntBetween(0, 5)) {
52 case 0:
53 base = randomAsciiLettersOfLength(('A' - separator) & 0xff);
54 break;
55
56 default:
57 base = randomAsciiLettersOfLengthBetween(1, 100);
58 break;
59 }
60
61 String inflected;
62 switch (randomIntBetween(0, 5)) {
63 case 0:
64 inflected = base;
65 break;
66
67 case 1:
68 inflected = randomAsciiLettersOfLengthBetween(0, 5) + base;
69 break;
70
71 case 3:
72 inflected = base + randomAsciiLettersOfLengthBetween(0, 5);
73 break;
74
75 case 4:
76 inflected = randomAsciiLettersOfLengthBetween(0, 5) + base + randomAsciiLettersOfLengthBetween(0, 5);
77 break;
78
79 default:
80 inflected = randomAsciiLettersOfLengthBetween(0, 200);
81 break;
82 }
83
84
85 sequences.add(
86 base + separator +
87 inflected +
88 (useTags ? (separator + randomAsciiLettersOfLengthBetween(0, 10)) : ""));
89 }
90
91 final boolean ignoreEmpty = randomBoolean();
92 try (Writer writer = Files.newBufferedWriter(input, StandardCharsets.UTF_8)) {
93 for (String in : sequences) {
94 writer.write(in);
95 writer.write('\n');
96
97 if (ignoreEmpty && randomBoolean()) {
98 writer.write('\n');
99 }
100 }
101 }
102
103 boolean validate = randomBoolean();
104 Assertions.assertThat(new DictCompile(input, false, validate, false, false, ignoreEmpty).call())
105 .isEqualTo(ExitStatus.SUCCESS);
106
107 Path dict = input.resolveSibling("dictionary.dict");
108 Assertions.assertThat(dict).isRegularFile();
109
110 // Verify the dictionary is valid.
111
112 DictionaryLookup dictionaryLookup = new DictionaryLookup(Dictionary.read(dict));
113 Set<String> reconstructed = new LinkedHashSet<>();
114 for (WordData wd : dictionaryLookup) {
115 reconstructed.add("" +
116 wd.getStem() + separator +
117 wd.getWord() +
118 (useTags ? separator : "") +
119 (wd.getTag() == null ? "" : wd.getTag()));
120 }
121
122 Assertions.assertThat(reconstructed).containsOnlyElementsOf(sequences);
123
124 // Verify decompilation via DictDecompile.
125
126 // GH-79: if there's only one sequence and there is no tag the decompiler will
127 // drop it.
128 if (useTags && sequences.size() == 1) {
129 String onlyOne = sequences.iterator().next();
130 if (onlyOne.endsWith(Character.toString(separator))) {
131 sequences.clear();
132 sequences.add(onlyOne.substring(0, onlyOne.length() - 1));
133 }
134 }
135
136 Files.delete(input);
137 Assertions.assertThat(new DictDecompile(dict, null, true, validate).call())
138 .isEqualTo(ExitStatus.SUCCESS);
139
140 List<String> allLines = Files.readAllLines(input, StandardCharsets.UTF_8);
141 Assertions.assertThat(allLines).containsOnlyElementsOf(sequences);
142 }
143 }