Debian Codegraph @

morfologik-tools/src/test/java/morfologik/tools/DictCompileBug.java

raw
1 package morfologik.tools;
2
3 import java.io.Writer;
4 import java.nio.charset.StandardCharsets;
5 import java.nio.file.Files;
6 import java.nio.file.Path;
7 import java.util.LinkedHashSet;
8 import java.util.Set;
9
10 import org.assertj.core.api.Assertions;
11 import org.junit.Test;
12
13 import com.carrotsearch.randomizedtesting.RandomizedTest;
14
15 import morfologik.stemming.Dictionary;
16 import morfologik.stemming.DictionaryLookup;
17 import morfologik.stemming.DictionaryMetadata;
18 import morfologik.stemming.EncoderType;
19 import morfologik.stemming.WordData;
20
21 public class DictCompileBug extends RandomizedTest {
22 @Test
23 public void testSeparatorInEncoded() throws Exception {
24 final Path input = newTempDir().resolve("dictionary.input");
25 final Path metadata = DictionaryMetadata.getExpectedMetadataLocation(input);
26
27 char separator = '_';
28 try (Writer writer = Files.newBufferedWriter(metadata, StandardCharsets.UTF_8)) {
29 DictionaryMetadata.builder()
30 .separator(separator)
31 .encoder(EncoderType.SUFFIX)
32 .encoding(StandardCharsets.UTF_8)
33 .build()
34 .write(writer);
35 }
36
37 Set<String> sequences = new LinkedHashSet<>();
38 for (int seqs = randomIntBetween(0, 100); --seqs >= 0;) {
39 sequences.add("anfragen_anfragen|VER:1:PLU:KJ1:SFT:NEB");
40 sequences.add("Anfragen_anfragen|VER:1:PLU:KJ1:SFT:NEB");
41 }
42
43 try (Writer writer = Files.newBufferedWriter(input, StandardCharsets.UTF_8)) {
44 for (String in : sequences) {
45 writer.write(in);
46 writer.write('\n');
47 }
48 }
49
50 Assertions.assertThat(new DictCompile(input, false, true, false, false, false).call())
51 .isEqualTo(ExitStatus.SUCCESS);
52
53 Path dict = input.resolveSibling("dictionary.dict");
54 Assertions.assertThat(dict).isRegularFile();
55
56 // Verify the dictionary is valid.
57
58 DictionaryLookup dictionaryLookup = new DictionaryLookup(Dictionary.read(dict));
59 for (WordData wd : dictionaryLookup) {
60 System.out.println(wd);
61 }
62 }
63 }