Debian Codegraph @

morfologik-fsa-builders/src/test/java/morfologik/fsa/builders/SerializerTestBase.java

raw
1 package morfologik.fsa.builders;
2
3 import static morfologik.fsa.FSAFlags.*;
4 import static org.junit.Assert.*;
5
6 import java.io.ByteArrayInputStream;
7 import java.io.ByteArrayOutputStream;
8 import java.io.IOException;
9 import java.nio.ByteBuffer;
10 import java.util.ArrayList;
11 import java.util.Arrays;
12 import java.util.Collections;
13 import java.util.HashSet;
14
15 import morfologik.fsa.FSA;
16 import morfologik.fsa.FSAFlags;
17
18 import org.junit.Test;
19
20 import static com.carrotsearch.randomizedtesting.RandomizedTest.*;
21
22 public abstract class SerializerTestBase extends TestBase {
23 @Test
24 public void testA() throws IOException {
25 byte[][] input = new byte[][] { { 'a' }, };
26
27 Arrays.sort(input, FSABuilder.LEXICAL_ORDERING);
28 FSA s = FSABuilder.build(input);
29
30 checkSerialization(input, s);
31 }
32
33 @Test
34 public void testArcsSharing() throws IOException {
35 byte[][] input = new byte[][] {
36 { 'a', 'c', 'f' },
37 { 'a', 'd', 'g' },
38 { 'a', 'e', 'h' },
39 { 'b', 'd', 'g' },
40 { 'b', 'e', 'h' },
41 };
42
43 Arrays.sort(input, FSABuilder.LEXICAL_ORDERING);
44 FSA s = FSABuilder.build(input);
45
46 checkSerialization(input, s);
47 }
48
49 @Test
50 public void testFSA5SerializerSimple() throws IOException {
51 byte[][] input = new byte[][] {
52 { 'a' },
53 { 'a', 'b', 'a' },
54 { 'a', 'c' },
55 { 'b' },
56 { 'b', 'a' },
57 { 'c' },
58 };
59
60 Arrays.sort(input, FSABuilder.LEXICAL_ORDERING);
61 FSA s = FSABuilder.build(input);
62
63 checkSerialization(input, s);
64 }
65
66 @Test
67 public void testNotMinimal() throws IOException {
68 byte[][] input = new byte[][] {
69 { 'a', 'b', 'a' },
70 { 'b' },
71 { 'b', 'a' }
72 };
73
74 Arrays.sort(input, FSABuilder.LEXICAL_ORDERING);
75 FSA s = FSABuilder.build(input);
76
77 checkSerialization(input, s);
78 }
79
80 @Test
81 public void testFSA5Bug0() throws IOException {
82 checkCorrect(new String[] {
83 "3-D+A+JJ",
84 "3-D+A+NN",
85 "4-F+A+NN",
86 "z+A+NN", });
87 }
88
89 @Test
90 public void testFSA5Bug1() throws IOException {
91 checkCorrect(new String[] { "+NP", "n+N", "n+NP", });
92 }
93
94 private void checkCorrect(String[] strings) throws IOException {
95 byte[][] input = new byte[strings.length][];
96 for (int i = 0; i < strings.length; i++) {
97 input[i] = strings[i].getBytes("ISO8859-1");
98 }
99
100 Arrays.sort(input, FSABuilder.LEXICAL_ORDERING);
101 FSA s = FSABuilder.build(input);
102
103 checkSerialization(input, s);
104 }
105
106 @Test
107 public void testEmptyInput() throws IOException {
108 byte[][] input = new byte[][] {};
109 FSA s = FSABuilder.build(input);
110
111 checkSerialization(input, s);
112 }
113
114 @Test
115 public void test_abc() throws IOException {
116 testBuiltIn(FSA.read(FSA5Test.class.getResourceAsStream("abc.fsa")));
117 }
118
119 @Test
120 public void test_minimal() throws IOException {
121 testBuiltIn(FSA.read(FSA5Test.class.getResourceAsStream("minimal.fsa")));
122 }
123
124 @Test
125 public void test_minimal2() throws IOException {
126 testBuiltIn(FSA.read(FSA5Test.class.getResourceAsStream("minimal2.fsa")));
127 }
128
129 @Test
130 public void test_en_tst() throws IOException {
131 testBuiltIn(FSA.read(FSA5Test.class.getResourceAsStream("en_tst.dict")));
132 }
133
134 private void testBuiltIn(FSA fsa) throws IOException {
135 final ArrayList<byte[]> sequences = new ArrayList<byte[]>();
136
137 sequences.clear();
138 for (ByteBuffer bb : fsa) {
139 sequences.add(Arrays.copyOf(bb.array(), bb.remaining()));
140 }
141
142 Collections.sort(sequences, FSABuilder.LEXICAL_ORDERING);
143
144 final byte[][] in = sequences.toArray(new byte[sequences.size()][]);
145 FSA root = FSABuilder.build(in);
146
147 // Check if the DFSA is correct first.
148 FSATestUtils.checkCorrect(in, root);
149
150 // Check serialization.
151 checkSerialization(in, root);
152 }
153
154 private void checkSerialization(byte[][] input, FSA root) throws IOException {
155 checkSerialization0(createSerializer(), input, root);
156 if (createSerializer().getFlags().contains(FSAFlags.NUMBERS)) {
157 checkSerialization0(createSerializer().withNumbers(), input, root);
158 }
159 }
160
161 private void checkSerialization0(FSASerializer serializer, final byte[][] in, FSA root) throws IOException {
162 final byte[] fsaData = serializer.serialize(root, new ByteArrayOutputStream()).toByteArray();
163
164 FSA fsa = FSA.read(new ByteArrayInputStream(fsaData));
165 checkCorrect(in, fsa);
166 }
167
168 /*
169 * Check if the FSA is correct with respect to the given input.
170 */
171 protected void checkCorrect(byte[][] input, FSA fsa) {
172 // (1) All input sequences are in the right language.
173 HashSet<ByteBuffer> rl = new HashSet<ByteBuffer>();
174 for (ByteBuffer bb : fsa) {
175 byte[] array = bb.array();
176 int length = bb.remaining();
177 rl.add(ByteBuffer.wrap(Arrays.copyOf(array, length)));
178 }
179
180 HashSet<ByteBuffer> uniqueInput = new HashSet<ByteBuffer>();
181 for (byte[] sequence : input) {
182 uniqueInput.add(ByteBuffer.wrap(sequence));
183 }
184
185 for (ByteBuffer sequence : uniqueInput) {
186 if (!rl.remove(sequence)) {
187 fail("Not present in the right language: " + toString(sequence));
188 }
189 }
190
191 // (2) No other sequence _other_ than the input is in the right
192 // language.
193 assertEquals(0, rl.size());
194 }
195
196 @Test
197 public void testAutomatonWithNodeNumbers() throws IOException {
198 assumeTrue(createSerializer().getFlags().contains(FSAFlags.NUMBERS));
199
200 byte[][] input = new byte[][] {
201 { 'a' },
202 { 'a', 'b', 'a' },
203 { 'a', 'c' },
204 { 'b' },
205 { 'b', 'a' },
206 { 'c' }, };
207
208 Arrays.sort(input, FSABuilder.LEXICAL_ORDERING);
209 FSA s = FSABuilder.build(input);
210
211 final byte[] fsaData =
212 createSerializer().withNumbers()
213 .serialize(s, new ByteArrayOutputStream())
214 .toByteArray();
215
216 FSA fsa = FSA.read(new ByteArrayInputStream(fsaData));
217
218 // Ensure we have the NUMBERS flag set.
219 assertTrue(fsa.getFlags().contains(NUMBERS));
220
221 // Get all numbers from nodes.
222 byte[] buffer = new byte[128];
223 final ArrayList<String> result = new ArrayList<String>();
224 FSA5Test.walkNode(buffer, 0, fsa, fsa.getRootNode(), 0, result);
225
226 Collections.sort(result);
227 assertEquals(
228 Arrays.asList("0 a", "1 aba", "2 ac", "3 b", "4 ba", "5 c"),
229 result);
230 }
231
232 protected abstract FSASerializer createSerializer();
233
234 /*
235 * Drain bytes from a byte buffer to a string.
236 */
237 public static String toString(ByteBuffer sequence) {
238 byte [] bytes = new byte [sequence.remaining()];
239 sequence.get(bytes);
240 return Arrays.toString(bytes);
241 }
242 }