| 1 |
package morfologik.tools; |
| 2 |
|
| 3 |
import java.io.Writer; |
| 4 |
import java.nio.charset.StandardCharsets; |
| 5 |
import java.nio.file.Files; |
| 6 |
import java.nio.file.Path; |
| 7 |
import java.util.LinkedHashSet; |
| 8 |
import java.util.List; |
| 9 |
import java.util.Set; |
| 10 |
|
| 11 |
import morfologik.stemming.Dictionary; |
| 12 |
import morfologik.stemming.DictionaryLookup; |
| 13 |
import morfologik.stemming.DictionaryMetadata; |
| 14 |
import morfologik.stemming.EncoderType; |
| 15 |
import morfologik.stemming.WordData; |
| 16 |
|
| 17 |
import org.assertj.core.api.Assertions; |
| 18 |
import org.junit.Test; |
| 19 |
|
| 20 |
import com.carrotsearch.randomizedtesting.RandomizedTest; |
| 21 |
import com.carrotsearch.randomizedtesting.annotations.Repeat; |
| 22 |
import com.carrotsearch.randomizedtesting.generators.RandomPicks; |
| 23 |
|
| 24 |
public class DictCompileTest extends RandomizedTest { |
| 25 |
@Test |
| 26 |
@Repeat(iterations = 200) |
| 27 |
public void testRoundTrip() throws Exception { |
| 28 |
final Path input = newTempDir().resolve("dictionary.input"); |
| 29 |
final Path metadata = DictionaryMetadata.getExpectedMetadataLocation(input); |
| 30 |
|
| 31 |
char separator = RandomPicks.randomFrom(getRandom(), new Character [] { |
| 32 |
'|', |
| 33 |
',', |
| 34 |
'\t', |
| 35 |
}); |
| 36 |
|
| 37 |
try (Writer writer = Files.newBufferedWriter(metadata, StandardCharsets.UTF_8)) { |
| 38 |
DictionaryMetadata.builder() |
| 39 |
.separator(separator) |
| 40 |
.encoder(randomFrom(EncoderType.values())) |
| 41 |
.encoding(StandardCharsets.UTF_8) |
| 42 |
.build() |
| 43 |
.write(writer); |
| 44 |
} |
| 45 |
|
| 46 |
final boolean useTags = randomBoolean(); |
| 47 |
|
| 48 |
Set<String> sequences = new LinkedHashSet<>(); |
| 49 |
for (int seqs = randomIntBetween(0, 100); --seqs >= 0;) { |
| 50 |
String base; |
| 51 |
switch (randomIntBetween(0, 5)) { |
| 52 |
case 0: |
| 53 |
base = randomAsciiLettersOfLength(('A' - separator) & 0xff); |
| 54 |
break; |
| 55 |
|
| 56 |
default: |
| 57 |
base = randomAsciiLettersOfLengthBetween(1, 100); |
| 58 |
break; |
| 59 |
} |
| 60 |
|
| 61 |
String inflected; |
| 62 |
switch (randomIntBetween(0, 5)) { |
| 63 |
case 0: |
| 64 |
inflected = base; |
| 65 |
break; |
| 66 |
|
| 67 |
case 1: |
| 68 |
inflected = randomAsciiLettersOfLengthBetween(0, 5) + base; |
| 69 |
break; |
| 70 |
|
| 71 |
case 3: |
| 72 |
inflected = base + randomAsciiLettersOfLengthBetween(0, 5); |
| 73 |
break; |
| 74 |
|
| 75 |
case 4: |
| 76 |
inflected = randomAsciiLettersOfLengthBetween(0, 5) + base + randomAsciiLettersOfLengthBetween(0, 5); |
| 77 |
break; |
| 78 |
|
| 79 |
default: |
| 80 |
inflected = randomAsciiLettersOfLengthBetween(0, 200); |
| 81 |
break; |
| 82 |
} |
| 83 |
|
| 84 |
|
| 85 |
sequences.add( |
| 86 |
base + separator + |
| 87 |
inflected + |
| 88 |
(useTags ? (separator + randomAsciiLettersOfLengthBetween(0, 10)) : "")); |
| 89 |
} |
| 90 |
|
| 91 |
final boolean ignoreEmpty = randomBoolean(); |
| 92 |
try (Writer writer = Files.newBufferedWriter(input, StandardCharsets.UTF_8)) { |
| 93 |
for (String in : sequences) { |
| 94 |
writer.write(in); |
| 95 |
writer.write('\n'); |
| 96 |
|
| 97 |
if (ignoreEmpty && randomBoolean()) { |
| 98 |
writer.write('\n'); |
| 99 |
} |
| 100 |
} |
| 101 |
} |
| 102 |
|
| 103 |
boolean validate = randomBoolean(); |
| 104 |
Assertions.assertThat(new DictCompile(input, false, validate, false, false, ignoreEmpty).call()) |
| 105 |
.isEqualTo(ExitStatus.SUCCESS); |
| 106 |
|
| 107 |
Path dict = input.resolveSibling("dictionary.dict"); |
| 108 |
Assertions.assertThat(dict).isRegularFile(); |
| 109 |
|
| 110 |
// Verify the dictionary is valid. |
| 111 |
|
| 112 |
DictionaryLookup dictionaryLookup = new DictionaryLookup(Dictionary.read(dict)); |
| 113 |
Set<String> reconstructed = new LinkedHashSet<>(); |
| 114 |
for (WordData wd : dictionaryLookup) { |
| 115 |
reconstructed.add("" + |
| 116 |
wd.getStem() + separator + |
| 117 |
wd.getWord() + |
| 118 |
(useTags ? separator : "") + |
| 119 |
(wd.getTag() == null ? "" : wd.getTag())); |
| 120 |
} |
| 121 |
|
| 122 |
Assertions.assertThat(reconstructed).containsOnlyElementsOf(sequences); |
| 123 |
|
| 124 |
// Verify decompilation via DictDecompile. |
| 125 |
|
| 126 |
// GH-79: if there's only one sequence and there is no tag the decompiler will |
| 127 |
// drop it. |
| 128 |
if (useTags && sequences.size() == 1) { |
| 129 |
String onlyOne = sequences.iterator().next(); |
| 130 |
if (onlyOne.endsWith(Character.toString(separator))) { |
| 131 |
sequences.clear(); |
| 132 |
sequences.add(onlyOne.substring(0, onlyOne.length() - 1)); |
| 133 |
} |
| 134 |
} |
| 135 |
|
| 136 |
Files.delete(input); |
| 137 |
Assertions.assertThat(new DictDecompile(dict, null, true, validate).call()) |
| 138 |
.isEqualTo(ExitStatus.SUCCESS); |
| 139 |
|
| 140 |
List<String> allLines = Files.readAllLines(input, StandardCharsets.UTF_8); |
| 141 |
Assertions.assertThat(allLines).containsOnlyElementsOf(sequences); |
| 142 |
} |
| 143 |
} |