| 1 |
package morfologik.tools; |
| 2 |
|
| 3 |
import java.io.Writer; |
| 4 |
import java.nio.charset.StandardCharsets; |
| 5 |
import java.nio.file.Files; |
| 6 |
import java.nio.file.Path; |
| 7 |
import java.util.LinkedHashSet; |
| 8 |
import java.util.Set; |
| 9 |
|
| 10 |
import org.assertj.core.api.Assertions; |
| 11 |
import org.junit.Test; |
| 12 |
|
| 13 |
import com.carrotsearch.randomizedtesting.RandomizedTest; |
| 14 |
|
| 15 |
import morfologik.stemming.Dictionary; |
| 16 |
import morfologik.stemming.DictionaryLookup; |
| 17 |
import morfologik.stemming.DictionaryMetadata; |
| 18 |
import morfologik.stemming.EncoderType; |
| 19 |
import morfologik.stemming.WordData; |
| 20 |
|
| 21 |
public class DictCompileBug extends RandomizedTest { |
| 22 |
@Test |
| 23 |
public void testSeparatorInEncoded() throws Exception { |
| 24 |
final Path input = newTempDir().resolve("dictionary.input"); |
| 25 |
final Path metadata = DictionaryMetadata.getExpectedMetadataLocation(input); |
| 26 |
|
| 27 |
char separator = '_'; |
| 28 |
try (Writer writer = Files.newBufferedWriter(metadata, StandardCharsets.UTF_8)) { |
| 29 |
DictionaryMetadata.builder() |
| 30 |
.separator(separator) |
| 31 |
.encoder(EncoderType.SUFFIX) |
| 32 |
.encoding(StandardCharsets.UTF_8) |
| 33 |
.build() |
| 34 |
.write(writer); |
| 35 |
} |
| 36 |
|
| 37 |
Set<String> sequences = new LinkedHashSet<>(); |
| 38 |
for (int seqs = randomIntBetween(0, 100); --seqs >= 0;) { |
| 39 |
sequences.add("anfragen_anfragen|VER:1:PLU:KJ1:SFT:NEB"); |
| 40 |
sequences.add("Anfragen_anfragen|VER:1:PLU:KJ1:SFT:NEB"); |
| 41 |
} |
| 42 |
|
| 43 |
try (Writer writer = Files.newBufferedWriter(input, StandardCharsets.UTF_8)) { |
| 44 |
for (String in : sequences) { |
| 45 |
writer.write(in); |
| 46 |
writer.write('\n'); |
| 47 |
} |
| 48 |
} |
| 49 |
|
| 50 |
Assertions.assertThat(new DictCompile(input, false, true, false, false, false).call()) |
| 51 |
.isEqualTo(ExitStatus.SUCCESS); |
| 52 |
|
| 53 |
Path dict = input.resolveSibling("dictionary.dict"); |
| 54 |
Assertions.assertThat(dict).isRegularFile(); |
| 55 |
|
| 56 |
// Verify the dictionary is valid. |
| 57 |
|
| 58 |
DictionaryLookup dictionaryLookup = new DictionaryLookup(Dictionary.read(dict)); |
| 59 |
for (WordData wd : dictionaryLookup) { |
| 60 |
System.out.println(wd); |
| 61 |
} |
| 62 |
} |
| 63 |
} |