Debian Codegraph @

morfologik-tools/src/test/java/morfologik/tools/FSACompileTest.java

raw
1 package morfologik.tools;
2
3 import java.io.ByteArrayOutputStream;
4 import java.io.InputStream;
5 import java.io.OutputStream;
6 import java.io.PrintStream;
7 import java.nio.ByteBuffer;
8 import java.nio.charset.StandardCharsets;
9 import java.nio.file.Files;
10 import java.nio.file.Path;
11 import java.util.HashSet;
12 import java.util.Iterator;
13 import java.util.LinkedHashSet;
14 import java.util.Set;
15 import java.util.concurrent.Callable;
16
17 import morfologik.fsa.FSA;
18 import morfologik.stemming.BufferUtils;
19
20 import org.assertj.core.api.Assertions;
21 import org.junit.Test;
22
23 import com.carrotsearch.randomizedtesting.RandomizedTest;
24 import com.carrotsearch.randomizedtesting.annotations.Repeat;
25
26 public class FSACompileTest extends RandomizedTest {
27 @Test
28 @Repeat(iterations = 100)
29 public void testCliInvocation() throws Exception {
30 final Path input = newTempFile();
31 final Path output = newTempFile();
32
33 Set<String> sequences = new LinkedHashSet<>();
34 for (int seqs = randomIntBetween(0, 100); --seqs >= 0;) {
35 sequences.add(randomAsciiLettersOfLengthBetween(1, 10));
36 }
37
38 try (OutputStream os = Files.newOutputStream(input)) {
39 Iterator<String> i = sequences.iterator();
40 while (i.hasNext()) {
41 os.write(i.next().getBytes(StandardCharsets.UTF_8));
42
43 // Sometimes don't add trailing '\n'.
44 if (!i.hasNext() && randomBoolean()) {
45 break;
46 } else {
47 os.write('\n');
48 if (randomBoolean()) {
49 os.write('\n');
50 }
51 }
52 }
53 }
54
55 SerializationFormat format = randomFrom(SerializationFormat.values());
56
57 Assertions.assertThat(new FSACompile(
58 input,
59 output,
60 format,
61 false,
62 false,
63 true).call()).isEqualTo(ExitStatus.SUCCESS);
64
65 try (InputStream is = Files.newInputStream(output)) {
66 FSA fsa = FSA.read(is);
67 Assertions.assertThat(fsa).isNotNull();
68
69 Set<String> result = new HashSet<>();
70 for (ByteBuffer bb : fsa) {
71 result.add(BufferUtils.toString(bb, StandardCharsets.UTF_8));
72 }
73
74 Assertions.assertThat(result).containsOnlyElementsOf(sequences);
75 }
76 }
77
78 @Test
79 public void testEmptyWarning() throws Exception {
80 final Path input = newTempFile();
81 final Path output = newTempFile();
82
83 Files.write(input, "abc\n\ndef".getBytes(StandardCharsets.US_ASCII));
84
85 String out = sysouts(new Callable<Void>() {
86 @Override
87 public Void call() throws Exception {
88 FSACompile.main(new String[] {
89 "--exit", "false",
90 "--input", input.toAbsolutePath().toString(),
91 "--output", output.toAbsolutePath().toString() });
92 return null;
93 }
94 });
95
96 Assertions.assertThat(out).contains("--ignore-empty");
97 }
98
99 @Test
100 public void testCrWarning() throws Exception {
101 final Path input = newTempFile();
102 final Path output = newTempFile();
103
104 Files.write(input, "abc\r\ndef\r\n".getBytes(StandardCharsets.US_ASCII));
105
106 String out = sysouts(new Callable<Void>() {
107 @Override
108 public Void call() throws Exception {
109 FSACompile.main(new String[] {
110 "--exit", "false",
111 "--input", input.toAbsolutePath().toString(),
112 "--output", output.toAbsolutePath().toString() });
113 return null;
114 }
115 });
116
117 Assertions.assertThat(out).contains("CR");
118 }
119
120 @Test
121 public void testBomWarning() throws Exception {
122 final Path input = newTempFile();
123 final Path output = newTempFile();
124
125 // Emit UTF-8 BOM prefixed list of three strings.
126 ByteArrayOutputStream baos = new ByteArrayOutputStream();
127 baos.write(new byte[] { (byte) 0xEF, (byte) 0xBB, (byte) 0xBF });
128 baos.write("abc\ndef\nxyz".getBytes(UTF8));
129 Files.write(input, baos.toByteArray());
130
131 String out = sysouts(new Callable<Void>() {
132 @Override
133 public Void call() throws Exception {
134 FSACompile.main(new String[] {
135 "--exit", "false",
136 "--input", input.toAbsolutePath().toString(),
137 "--output", output.toAbsolutePath().toString() });
138 return null;
139 }
140 });
141
142 Assertions.assertThat(out).contains("UTF-8 BOM");
143 }
144
145 private String sysouts(Callable<Void> callable) throws Exception {
146 PrintStream sout = System.out;
147 PrintStream serr = System.err;
148
149 ByteArrayOutputStream baos = new ByteArrayOutputStream();
150 PrintStream ps = new PrintStream(baos, true, "UTF-8");
151 System.setOut(ps);
152 System.setErr(ps);
153 try {
154 callable.call();
155 return new String(baos.toByteArray(), StandardCharsets.UTF_8);
156 } finally {
157 System.setOut(sout);
158 System.setErr(serr);
159 }
160 }
161 }