| 1 |
// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT |
| 2 |
// file at the top-level directory of this distribution and at |
| 3 |
// http://rust-lang.org/COPYRIGHT. |
| 4 |
// |
| 5 |
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or |
| 6 |
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license |
| 7 |
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your |
| 8 |
// option. This file may not be copied, modified, or distributed |
| 9 |
// except according to those terms. |
| 10 |
|
| 11 |
use super::UnicodeSegmentation; |
| 12 |
|
| 13 |
use std::prelude::v1::*; |
| 14 |
|
| 15 |
#[test] |
| 16 |
fn test_graphemes() { |
| 17 |
use crate::testdata::{TEST_DIFF, TEST_SAME}; |
| 18 |
|
| 19 |
pub const EXTRA_DIFF: &'static [( |
| 20 |
&'static str, |
| 21 |
&'static [&'static str], |
| 22 |
&'static [&'static str], |
| 23 |
)] = &[ |
| 24 |
// Official test suite doesn't include two Prepend chars between two other chars. |
| 25 |
( |
| 26 |
"\u{20}\u{600}\u{600}\u{20}", |
| 27 |
&["\u{20}", "\u{600}\u{600}\u{20}"], |
| 28 |
&["\u{20}", "\u{600}", "\u{600}", "\u{20}"], |
| 29 |
), |
| 30 |
// Test for Prepend followed by two Any chars |
| 31 |
( |
| 32 |
"\u{600}\u{20}\u{20}", |
| 33 |
&["\u{600}\u{20}", "\u{20}"], |
| 34 |
&["\u{600}", "\u{20}", "\u{20}"], |
| 35 |
), |
| 36 |
]; |
| 37 |
|
| 38 |
pub const EXTRA_SAME: &'static [(&'static str, &'static [&'static str])] = &[ |
| 39 |
// family emoji (more than two emoji joined by ZWJ) |
| 40 |
( |
| 41 |
"\u{1f468}\u{200d}\u{1f467}\u{200d}\u{1f466}", |
| 42 |
&["\u{1f468}\u{200d}\u{1f467}\u{200d}\u{1f466}"], |
| 43 |
), |
| 44 |
// cartwheel emoji followed by two fitzpatrick skin tone modifiers |
| 45 |
// (test case from issue #19) |
| 46 |
( |
| 47 |
"\u{1F938}\u{1F3FE}\u{1F3FE}", |
| 48 |
&["\u{1F938}\u{1F3FE}\u{1F3FE}"], |
| 49 |
), |
| 50 |
]; |
| 51 |
|
| 52 |
for &(s, g) in TEST_SAME.iter().chain(EXTRA_SAME) { |
| 53 |
if s.starts_with("क\u{94d}") || s.starts_with("क\u{93c}") { |
| 54 |
continue; // TODO: fix these |
| 55 |
} |
| 56 |
// test forward iterator |
| 57 |
assert!(UnicodeSegmentation::graphemes(s, true).eq(g.iter().cloned())); |
| 58 |
assert!(UnicodeSegmentation::graphemes(s, false).eq(g.iter().cloned())); |
| 59 |
|
| 60 |
// test reverse iterator |
| 61 |
assert!(UnicodeSegmentation::graphemes(s, true) |
| 62 |
.rev() |
| 63 |
.eq(g.iter().rev().cloned())); |
| 64 |
assert!(UnicodeSegmentation::graphemes(s, false) |
| 65 |
.rev() |
| 66 |
.eq(g.iter().rev().cloned())); |
| 67 |
} |
| 68 |
|
| 69 |
for &(s, gt, gf) in TEST_DIFF.iter().chain(EXTRA_DIFF) { |
| 70 |
// test forward iterator |
| 71 |
assert!(UnicodeSegmentation::graphemes(s, true).eq(gt.iter().cloned())); |
| 72 |
assert!(UnicodeSegmentation::graphemes(s, false).eq(gf.iter().cloned())); |
| 73 |
|
| 74 |
// test reverse iterator |
| 75 |
assert!(UnicodeSegmentation::graphemes(s, true) |
| 76 |
.rev() |
| 77 |
.eq(gt.iter().rev().cloned())); |
| 78 |
assert!(UnicodeSegmentation::graphemes(s, false) |
| 79 |
.rev() |
| 80 |
.eq(gf.iter().rev().cloned())); |
| 81 |
} |
| 82 |
|
| 83 |
// test the indices iterators |
| 84 |
let s = "a̐éö̲\r\n"; |
| 85 |
let gr_inds = UnicodeSegmentation::grapheme_indices(s, true).collect::<Vec<(usize, &str)>>(); |
| 86 |
let b: &[_] = &[(0, "a̐"), (3, "é"), (6, "ö̲"), (11, "\r\n")]; |
| 87 |
assert_eq!(gr_inds, b); |
| 88 |
let gr_inds = UnicodeSegmentation::grapheme_indices(s, true) |
| 89 |
.rev() |
| 90 |
.collect::<Vec<(usize, &str)>>(); |
| 91 |
let b: &[_] = &[(11, "\r\n"), (6, "ö̲"), (3, "é"), (0, "a̐")]; |
| 92 |
assert_eq!(gr_inds, b); |
| 93 |
let mut gr_inds_iter = UnicodeSegmentation::grapheme_indices(s, true); |
| 94 |
{ |
| 95 |
let gr_inds = gr_inds_iter.by_ref(); |
| 96 |
let e1 = gr_inds.size_hint(); |
| 97 |
assert_eq!(e1, (1, Some(13))); |
| 98 |
let c = gr_inds.count(); |
| 99 |
assert_eq!(c, 4); |
| 100 |
} |
| 101 |
let e2 = gr_inds_iter.size_hint(); |
| 102 |
assert_eq!(e2, (0, Some(0))); |
| 103 |
|
| 104 |
// make sure the reverse iterator does the right thing with "\n" at beginning of string |
| 105 |
let s = "\n\r\n\r"; |
| 106 |
let gr = UnicodeSegmentation::graphemes(s, true) |
| 107 |
.rev() |
| 108 |
.collect::<Vec<&str>>(); |
| 109 |
let b: &[_] = &["\r", "\r\n", "\n"]; |
| 110 |
assert_eq!(gr, b); |
| 111 |
} |
| 112 |
|
| 113 |
#[test] |
| 114 |
fn test_words() { |
| 115 |
use crate::testdata::TEST_WORD; |
| 116 |
|
| 117 |
// Unicode's official tests don't really test longer chains of flag emoji |
| 118 |
// TODO This could be improved with more tests like flag emoji with interspersed Extend chars and ZWJ |
| 119 |
const EXTRA_TESTS: &'static [(&'static str, &'static [&'static str])] = &[ |
| 120 |
( |
| 121 |
"🇦🇫🇦🇽🇦🇱🇩🇿🇦🇸🇦🇩🇦🇴", |
| 122 |
&["🇦🇫", "🇦🇽", "🇦🇱", "🇩🇿", "🇦🇸", "🇦🇩", "🇦🇴"], |
| 123 |
), |
| 124 |
("🇦🇫🇦🇽🇦🇱🇩🇿🇦🇸🇦🇩🇦", &["🇦🇫", "🇦🇽", "🇦🇱", "🇩🇿", "🇦🇸", "🇦🇩", "🇦"]), |
| 125 |
( |
| 126 |
"🇦a🇫🇦🇽a🇦🇱🇩🇿🇦🇸🇦🇩🇦", |
| 127 |
&["🇦", "a", "🇫🇦", "🇽", "a", "🇦🇱", "🇩🇿", "🇦🇸", "🇦🇩", "🇦"], |
| 128 |
), |
| 129 |
( |
| 130 |
"\u{1f468}\u{200d}\u{1f468}\u{200d}\u{1f466}", |
| 131 |
&["\u{1f468}\u{200d}\u{1f468}\u{200d}\u{1f466}"], |
| 132 |
), |
| 133 |
("😌👎🏼", &["😌", "👎🏼"]), |
| 134 |
// perhaps wrong, spaces should not be included? |
| 135 |
("hello world", &["hello", " ", "world"]), |
| 136 |
("🇨🇦🇨🇭🇿🇲🇿 hi", &["🇨🇦", "🇨🇭", "🇿🇲", "🇿", " ", "hi"]), |
| 137 |
]; |
| 138 |
for &(s, w) in TEST_WORD.iter().chain(EXTRA_TESTS.iter()) { |
| 139 |
macro_rules! assert_ { |
| 140 |
($test:expr, $exp:expr, $name:expr) => { |
| 141 |
// collect into vector for better diagnostics in failure case |
| 142 |
let testing = $test.collect::<Vec<_>>(); |
| 143 |
let expected = $exp.collect::<Vec<_>>(); |
| 144 |
assert_eq!( |
| 145 |
testing, expected, |
| 146 |
"{} test for testcase ({:?}, {:?}) failed.", |
| 147 |
$name, s, w |
| 148 |
) |
| 149 |
}; |
| 150 |
} |
| 151 |
// test forward iterator |
| 152 |
assert_!( |
| 153 |
s.split_word_bounds(), |
| 154 |
w.iter().cloned(), |
| 155 |
"Forward word boundaries" |
| 156 |
); |
| 157 |
|
| 158 |
// test reverse iterator |
| 159 |
assert_!( |
| 160 |
s.split_word_bounds().rev(), |
| 161 |
w.iter().rev().cloned(), |
| 162 |
"Reverse word boundaries" |
| 163 |
); |
| 164 |
|
| 165 |
// generate offsets from word string lengths |
| 166 |
let mut indices = vec![0]; |
| 167 |
for i in w.iter().cloned().map(|s| s.len()).scan(0, |t, n| { |
| 168 |
*t += n; |
| 169 |
Some(*t) |
| 170 |
}) { |
| 171 |
indices.push(i); |
| 172 |
} |
| 173 |
indices.pop(); |
| 174 |
let indices = indices; |
| 175 |
|
| 176 |
// test forward indices iterator |
| 177 |
assert_!( |
| 178 |
s.split_word_bound_indices().map(|(l, _)| l), |
| 179 |
indices.iter().cloned(), |
| 180 |
"Forward word indices" |
| 181 |
); |
| 182 |
|
| 183 |
// test backward indices iterator |
| 184 |
assert_!( |
| 185 |
s.split_word_bound_indices().rev().map(|(l, _)| l), |
| 186 |
indices.iter().rev().cloned(), |
| 187 |
"Reverse word indices" |
| 188 |
); |
| 189 |
} |
| 190 |
} |
| 191 |
|
| 192 |
#[test] |
| 193 |
fn test_sentences() { |
| 194 |
use crate::testdata::TEST_SENTENCE; |
| 195 |
|
| 196 |
for &(s, w) in TEST_SENTENCE.iter() { |
| 197 |
macro_rules! assert_ { |
| 198 |
($test:expr, $exp:expr, $name:expr) => { |
| 199 |
// collect into vector for better diagnostics in failure case |
| 200 |
let testing = $test.collect::<Vec<_>>(); |
| 201 |
let expected = $exp.collect::<Vec<_>>(); |
| 202 |
assert_eq!( |
| 203 |
testing, expected, |
| 204 |
"{} test for testcase ({:?}, {:?}) failed.", |
| 205 |
$name, s, w |
| 206 |
) |
| 207 |
}; |
| 208 |
} |
| 209 |
|
| 210 |
assert_!( |
| 211 |
s.split_sentence_bounds(), |
| 212 |
w.iter().cloned(), |
| 213 |
"Forward sentence boundaries" |
| 214 |
); |
| 215 |
} |
| 216 |
} |
| 217 |
|
| 218 |
#[test] |
| 219 |
fn test_syriac_abbr_mark() { |
| 220 |
use crate::tables::word as wd; |
| 221 |
let (_, _, cat) = wd::word_category('\u{70f}'); |
| 222 |
assert_eq!(cat, wd::WC_ALetter); |
| 223 |
} |
| 224 |
|
| 225 |
#[test] |
| 226 |
fn test_end_of_ayah_cat() { |
| 227 |
use crate::tables::word as wd; |
| 228 |
let (_, _, cat) = wd::word_category('\u{6dd}'); |
| 229 |
assert_eq!(cat, wd::WC_Numeric); |
| 230 |
} |
| 231 |
|
| 232 |
quickcheck! { |
| 233 |
fn quickcheck_forward_reverse_graphemes_extended(s: String) -> bool { |
| 234 |
let a = s.graphemes(true).collect::<Vec<_>>(); |
| 235 |
let mut b = s.graphemes(true).rev().collect::<Vec<_>>(); |
| 236 |
b.reverse(); |
| 237 |
a == b |
| 238 |
} |
| 239 |
|
| 240 |
fn quickcheck_forward_reverse_graphemes_legacy(s: String) -> bool { |
| 241 |
let a = s.graphemes(false).collect::<Vec<_>>(); |
| 242 |
let mut b = s.graphemes(false).rev().collect::<Vec<_>>(); |
| 243 |
b.reverse(); |
| 244 |
a == b |
| 245 |
} |
| 246 |
|
| 247 |
fn quickcheck_join_graphemes(s: String) -> bool { |
| 248 |
let a = s.graphemes(true).collect::<String>(); |
| 249 |
let b = s.graphemes(false).collect::<String>(); |
| 250 |
a == s && b == s |
| 251 |
} |
| 252 |
|
| 253 |
fn quickcheck_forward_reverse_words(s: String) -> bool { |
| 254 |
let a = s.split_word_bounds().collect::<Vec<_>>(); |
| 255 |
let mut b = s.split_word_bounds().rev().collect::<Vec<_>>(); |
| 256 |
b.reverse(); |
| 257 |
a == b |
| 258 |
} |
| 259 |
|
| 260 |
fn quickcheck_join_words(s: String) -> bool { |
| 261 |
let a = s.split_word_bounds().collect::<String>(); |
| 262 |
a == s |
| 263 |
} |
| 264 |
} |