Debian Codegraph @

.cargo/vendor/unicode-segmentation/src/test.rs

raw
1 // Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
4 //
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
10
11 use super::UnicodeSegmentation;
12
13 use std::prelude::v1::*;
14
15 #[test]
16 fn test_graphemes() {
17 use crate::testdata::{TEST_DIFF, TEST_SAME};
18
19 pub const EXTRA_DIFF: &'static [(
20 &'static str,
21 &'static [&'static str],
22 &'static [&'static str],
23 )] = &[
24 // Official test suite doesn't include two Prepend chars between two other chars.
25 (
26 "\u{20}\u{600}\u{600}\u{20}",
27 &["\u{20}", "\u{600}\u{600}\u{20}"],
28 &["\u{20}", "\u{600}", "\u{600}", "\u{20}"],
29 ),
30 // Test for Prepend followed by two Any chars
31 (
32 "\u{600}\u{20}\u{20}",
33 &["\u{600}\u{20}", "\u{20}"],
34 &["\u{600}", "\u{20}", "\u{20}"],
35 ),
36 ];
37
38 pub const EXTRA_SAME: &'static [(&'static str, &'static [&'static str])] = &[
39 // family emoji (more than two emoji joined by ZWJ)
40 (
41 "\u{1f468}\u{200d}\u{1f467}\u{200d}\u{1f466}",
42 &["\u{1f468}\u{200d}\u{1f467}\u{200d}\u{1f466}"],
43 ),
44 // cartwheel emoji followed by two fitzpatrick skin tone modifiers
45 // (test case from issue #19)
46 (
47 "\u{1F938}\u{1F3FE}\u{1F3FE}",
48 &["\u{1F938}\u{1F3FE}\u{1F3FE}"],
49 ),
50 ];
51
52 for &(s, g) in TEST_SAME.iter().chain(EXTRA_SAME) {
53 if s.starts_with("क\u{94d}") || s.starts_with("क\u{93c}") {
54 continue; // TODO: fix these
55 }
56 // test forward iterator
57 assert!(UnicodeSegmentation::graphemes(s, true).eq(g.iter().cloned()));
58 assert!(UnicodeSegmentation::graphemes(s, false).eq(g.iter().cloned()));
59
60 // test reverse iterator
61 assert!(UnicodeSegmentation::graphemes(s, true)
62 .rev()
63 .eq(g.iter().rev().cloned()));
64 assert!(UnicodeSegmentation::graphemes(s, false)
65 .rev()
66 .eq(g.iter().rev().cloned()));
67 }
68
69 for &(s, gt, gf) in TEST_DIFF.iter().chain(EXTRA_DIFF) {
70 // test forward iterator
71 assert!(UnicodeSegmentation::graphemes(s, true).eq(gt.iter().cloned()));
72 assert!(UnicodeSegmentation::graphemes(s, false).eq(gf.iter().cloned()));
73
74 // test reverse iterator
75 assert!(UnicodeSegmentation::graphemes(s, true)
76 .rev()
77 .eq(gt.iter().rev().cloned()));
78 assert!(UnicodeSegmentation::graphemes(s, false)
79 .rev()
80 .eq(gf.iter().rev().cloned()));
81 }
82
83 // test the indices iterators
84 let s = "a̐éö̲\r\n";
85 let gr_inds = UnicodeSegmentation::grapheme_indices(s, true).collect::<Vec<(usize, &str)>>();
86 let b: &[_] = &[(0, "a̐"), (3, "é"), (6, "ö̲"), (11, "\r\n")];
87 assert_eq!(gr_inds, b);
88 let gr_inds = UnicodeSegmentation::grapheme_indices(s, true)
89 .rev()
90 .collect::<Vec<(usize, &str)>>();
91 let b: &[_] = &[(11, "\r\n"), (6, "ö̲"), (3, "é"), (0, "a̐")];
92 assert_eq!(gr_inds, b);
93 let mut gr_inds_iter = UnicodeSegmentation::grapheme_indices(s, true);
94 {
95 let gr_inds = gr_inds_iter.by_ref();
96 let e1 = gr_inds.size_hint();
97 assert_eq!(e1, (1, Some(13)));
98 let c = gr_inds.count();
99 assert_eq!(c, 4);
100 }
101 let e2 = gr_inds_iter.size_hint();
102 assert_eq!(e2, (0, Some(0)));
103
104 // make sure the reverse iterator does the right thing with "\n" at beginning of string
105 let s = "\n\r\n\r";
106 let gr = UnicodeSegmentation::graphemes(s, true)
107 .rev()
108 .collect::<Vec<&str>>();
109 let b: &[_] = &["\r", "\r\n", "\n"];
110 assert_eq!(gr, b);
111 }
112
113 #[test]
114 fn test_words() {
115 use crate::testdata::TEST_WORD;
116
117 // Unicode's official tests don't really test longer chains of flag emoji
118 // TODO This could be improved with more tests like flag emoji with interspersed Extend chars and ZWJ
119 const EXTRA_TESTS: &'static [(&'static str, &'static [&'static str])] = &[
120 (
121 "🇦🇫🇦🇽🇦🇱🇩🇿🇦🇸🇦🇩🇦🇴",
122 &["🇦🇫", "🇦🇽", "🇦🇱", "🇩🇿", "🇦🇸", "🇦🇩", "🇦🇴"],
123 ),
124 ("🇦🇫🇦🇽🇦🇱🇩🇿🇦🇸🇦🇩🇦", &["🇦🇫", "🇦🇽", "🇦🇱", "🇩🇿", "🇦🇸", "🇦🇩", "🇦"]),
125 (
126 "🇦a🇫🇦🇽a🇦🇱🇩🇿🇦🇸🇦🇩🇦",
127 &["🇦", "a", "🇫🇦", "🇽", "a", "🇦🇱", "🇩🇿", "🇦🇸", "🇦🇩", "🇦"],
128 ),
129 (
130 "\u{1f468}\u{200d}\u{1f468}\u{200d}\u{1f466}",
131 &["\u{1f468}\u{200d}\u{1f468}\u{200d}\u{1f466}"],
132 ),
133 ("😌👎🏼", &["😌", "👎🏼"]),
134 // perhaps wrong, spaces should not be included?
135 ("hello world", &["hello", " ", "world"]),
136 ("🇨🇦🇨🇭🇿🇲🇿 hi", &["🇨🇦", "🇨🇭", "🇿🇲", "🇿", " ", "hi"]),
137 ];
138 for &(s, w) in TEST_WORD.iter().chain(EXTRA_TESTS.iter()) {
139 macro_rules! assert_ {
140 ($test:expr, $exp:expr, $name:expr) => {
141 // collect into vector for better diagnostics in failure case
142 let testing = $test.collect::<Vec<_>>();
143 let expected = $exp.collect::<Vec<_>>();
144 assert_eq!(
145 testing, expected,
146 "{} test for testcase ({:?}, {:?}) failed.",
147 $name, s, w
148 )
149 };
150 }
151 // test forward iterator
152 assert_!(
153 s.split_word_bounds(),
154 w.iter().cloned(),
155 "Forward word boundaries"
156 );
157
158 // test reverse iterator
159 assert_!(
160 s.split_word_bounds().rev(),
161 w.iter().rev().cloned(),
162 "Reverse word boundaries"
163 );
164
165 // generate offsets from word string lengths
166 let mut indices = vec![0];
167 for i in w.iter().cloned().map(|s| s.len()).scan(0, |t, n| {
168 *t += n;
169 Some(*t)
170 }) {
171 indices.push(i);
172 }
173 indices.pop();
174 let indices = indices;
175
176 // test forward indices iterator
177 assert_!(
178 s.split_word_bound_indices().map(|(l, _)| l),
179 indices.iter().cloned(),
180 "Forward word indices"
181 );
182
183 // test backward indices iterator
184 assert_!(
185 s.split_word_bound_indices().rev().map(|(l, _)| l),
186 indices.iter().rev().cloned(),
187 "Reverse word indices"
188 );
189 }
190 }
191
192 #[test]
193 fn test_sentences() {
194 use crate::testdata::TEST_SENTENCE;
195
196 for &(s, w) in TEST_SENTENCE.iter() {
197 macro_rules! assert_ {
198 ($test:expr, $exp:expr, $name:expr) => {
199 // collect into vector for better diagnostics in failure case
200 let testing = $test.collect::<Vec<_>>();
201 let expected = $exp.collect::<Vec<_>>();
202 assert_eq!(
203 testing, expected,
204 "{} test for testcase ({:?}, {:?}) failed.",
205 $name, s, w
206 )
207 };
208 }
209
210 assert_!(
211 s.split_sentence_bounds(),
212 w.iter().cloned(),
213 "Forward sentence boundaries"
214 );
215 }
216 }
217
218 #[test]
219 fn test_syriac_abbr_mark() {
220 use crate::tables::word as wd;
221 let (_, _, cat) = wd::word_category('\u{70f}');
222 assert_eq!(cat, wd::WC_ALetter);
223 }
224
225 #[test]
226 fn test_end_of_ayah_cat() {
227 use crate::tables::word as wd;
228 let (_, _, cat) = wd::word_category('\u{6dd}');
229 assert_eq!(cat, wd::WC_Numeric);
230 }
231
232 quickcheck! {
233 fn quickcheck_forward_reverse_graphemes_extended(s: String) -> bool {
234 let a = s.graphemes(true).collect::<Vec<_>>();
235 let mut b = s.graphemes(true).rev().collect::<Vec<_>>();
236 b.reverse();
237 a == b
238 }
239
240 fn quickcheck_forward_reverse_graphemes_legacy(s: String) -> bool {
241 let a = s.graphemes(false).collect::<Vec<_>>();
242 let mut b = s.graphemes(false).rev().collect::<Vec<_>>();
243 b.reverse();
244 a == b
245 }
246
247 fn quickcheck_join_graphemes(s: String) -> bool {
248 let a = s.graphemes(true).collect::<String>();
249 let b = s.graphemes(false).collect::<String>();
250 a == s && b == s
251 }
252
253 fn quickcheck_forward_reverse_words(s: String) -> bool {
254 let a = s.split_word_bounds().collect::<Vec<_>>();
255 let mut b = s.split_word_bounds().rev().collect::<Vec<_>>();
256 b.reverse();
257 a == b
258 }
259
260 fn quickcheck_join_words(s: String) -> bool {
261 let a = s.split_word_bounds().collect::<String>();
262 a == s
263 }
264 }