Debian Codegraph @

.cargo/vendor/unicode-segmentation/src/word.rs

raw
1 // Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
4 //
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
10
11 use core::cmp;
12 use core::iter::Filter;
13
14 use crate::tables::word::WordCat;
15
16 /// An iterator over the substrings of a string which, after splitting the string on
17 /// [word boundaries](http://www.unicode.org/reports/tr29/#Word_Boundaries),
18 /// contain any characters with the
19 /// [Alphabetic](http://unicode.org/reports/tr44/#Alphabetic)
20 /// property, or with
21 /// [General_Category=Number](http://unicode.org/reports/tr44/#General_Category_Values).
22 ///
23 /// This struct is created by the [`unicode_words`] method on the [`UnicodeSegmentation`] trait. See
24 /// its documentation for more.
25 ///
26 /// [`unicode_words`]: trait.UnicodeSegmentation.html#tymethod.unicode_words
27 /// [`UnicodeSegmentation`]: trait.UnicodeSegmentation.html
28 pub struct UnicodeWords<'a> {
29 inner: Filter<UWordBounds<'a>, fn(&&str) -> bool>,
30 }
31
32 impl<'a> Iterator for UnicodeWords<'a> {
33 type Item = &'a str;
34
35 #[inline]
36 fn next(&mut self) -> Option<&'a str> {
37 self.inner.next()
38 }
39
40 #[inline]
41 fn size_hint(&self) -> (usize, Option<usize>) {
42 self.inner.size_hint()
43 }
44 }
45 impl<'a> DoubleEndedIterator for UnicodeWords<'a> {
46 #[inline]
47 fn next_back(&mut self) -> Option<&'a str> {
48 self.inner.next_back()
49 }
50 }
51
52 /// An iterator over the substrings of a string which, after splitting the string on
53 /// [word boundaries](http://www.unicode.org/reports/tr29/#Word_Boundaries),
54 /// contain any characters with the
55 /// [Alphabetic](http://unicode.org/reports/tr44/#Alphabetic)
56 /// property, or with
57 /// [General_Category=Number](http://unicode.org/reports/tr44/#General_Category_Values).
58 /// This iterator also provides the byte offsets for each substring.
59 ///
60 /// This struct is created by the [`unicode_word_indices`] method on the [`UnicodeSegmentation`] trait. See
61 /// its documentation for more.
62 ///
63 /// [`unicode_word_indices`]: trait.UnicodeSegmentation.html#tymethod.unicode_word_indices
64 /// [`UnicodeSegmentation`]: trait.UnicodeSegmentation.html
65 pub struct UnicodeWordIndices<'a> {
66 inner: Filter<UWordBoundIndices<'a>, fn(&(usize, &str)) -> bool>,
67 }
68
69 impl<'a> Iterator for UnicodeWordIndices<'a> {
70 type Item = (usize, &'a str);
71
72 #[inline]
73 fn next(&mut self) -> Option<(usize, &'a str)> {
74 self.inner.next()
75 }
76
77 #[inline]
78 fn size_hint(&self) -> (usize, Option<usize>) {
79 self.inner.size_hint()
80 }
81 }
82 impl<'a> DoubleEndedIterator for UnicodeWordIndices<'a> {
83 #[inline]
84 fn next_back(&mut self) -> Option<(usize, &'a str)> {
85 self.inner.next_back()
86 }
87 }
88
89 /// External iterator for a string's
90 /// [word boundaries](http://www.unicode.org/reports/tr29/#Word_Boundaries).
91 ///
92 /// This struct is created by the [`split_word_bounds`] method on the [`UnicodeSegmentation`]
93 /// trait. See its documentation for more.
94 ///
95 /// [`split_word_bounds`]: trait.UnicodeSegmentation.html#tymethod.split_word_bounds
96 /// [`UnicodeSegmentation`]: trait.UnicodeSegmentation.html
97 #[derive(Clone)]
98 pub struct UWordBounds<'a> {
99 string: &'a str,
100 cat: Option<WordCat>,
101 catb: Option<WordCat>,
102 }
103
104 /// External iterator for word boundaries and byte offsets.
105 ///
106 /// This struct is created by the [`split_word_bound_indices`] method on the
107 /// [`UnicodeSegmentation`] trait. See its documentation for more.
108 ///
109 /// [`split_word_bound_indices`]: trait.UnicodeSegmentation.html#tymethod.split_word_bound_indices
110 /// [`UnicodeSegmentation`]: trait.UnicodeSegmentation.html
111 #[derive(Clone)]
112 pub struct UWordBoundIndices<'a> {
113 start_offset: usize,
114 iter: UWordBounds<'a>,
115 }
116
117 impl<'a> UWordBoundIndices<'a> {
118 #[inline]
119 /// View the underlying data (the part yet to be iterated) as a slice of the original string.
120 ///
121 /// ```rust
122 /// # use unicode_segmentation::UnicodeSegmentation;
123 /// let mut iter = "Hello world".split_word_bound_indices();
124 /// assert_eq!(iter.as_str(), "Hello world");
125 /// iter.next();
126 /// assert_eq!(iter.as_str(), " world");
127 /// iter.next();
128 /// assert_eq!(iter.as_str(), "world");
129 /// ```
130 pub fn as_str(&self) -> &'a str {
131 self.iter.as_str()
132 }
133 }
134
135 impl<'a> Iterator for UWordBoundIndices<'a> {
136 type Item = (usize, &'a str);
137
138 #[inline]
139 fn next(&mut self) -> Option<(usize, &'a str)> {
140 self.iter
141 .next()
142 .map(|s| (s.as_ptr() as usize - self.start_offset, s))
143 }
144
145 #[inline]
146 fn size_hint(&self) -> (usize, Option<usize>) {
147 self.iter.size_hint()
148 }
149 }
150
151 impl<'a> DoubleEndedIterator for UWordBoundIndices<'a> {
152 #[inline]
153 fn next_back(&mut self) -> Option<(usize, &'a str)> {
154 self.iter
155 .next_back()
156 .map(|s| (s.as_ptr() as usize - self.start_offset, s))
157 }
158 }
159
160 // state machine for word boundary rules
161 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
162 enum UWordBoundsState {
163 Start,
164 Letter,
165 HLetter,
166 Numeric,
167 Katakana,
168 ExtendNumLet,
169 Regional(RegionalState),
170 FormatExtend(FormatExtendType),
171 Zwj,
172 Emoji,
173 WSegSpace,
174 }
175
176 // subtypes for FormatExtend state in UWordBoundsState
177 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
178 enum FormatExtendType {
179 AcceptAny,
180 AcceptNone,
181 RequireLetter,
182 RequireHLetter,
183 AcceptQLetter,
184 RequireNumeric,
185 }
186
187 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
188 enum RegionalState {
189 Half,
190 Full,
191 Unknown,
192 }
193
194 fn is_emoji(ch: char) -> bool {
195 use crate::tables::emoji;
196 emoji::emoji_category(ch).2 == emoji::EmojiCat::EC_Extended_Pictographic
197 }
198
199 impl<'a> Iterator for UWordBounds<'a> {
200 type Item = &'a str;
201
202 #[inline]
203 fn size_hint(&self) -> (usize, Option<usize>) {
204 let slen = self.string.len();
205 (cmp::min(slen, 1), Some(slen))
206 }
207
208 #[inline]
209 fn next(&mut self) -> Option<&'a str> {
210 use self::FormatExtendType::*;
211 use self::UWordBoundsState::*;
212 use crate::tables::word as wd;
213 if self.string.len() == 0 {
214 return None;
215 }
216
217 let mut take_curr = true;
218 let mut take_cat = true;
219 let mut idx = 0;
220 let mut saveidx = 0;
221 let mut state = Start;
222 let mut cat = wd::WC_Any;
223 let mut savecat = wd::WC_Any;
224
225 // If extend/format/zwj were skipped. Handles precedence of WB3d over WB4
226 let mut skipped_format_extend = false;
227 for (curr, ch) in self.string.char_indices() {
228 idx = curr;
229 // Whether or not the previous category was ZWJ
230 // ZWJs get collapsed, so this handles precedence of WB3c over WB4
231 let prev_zwj = cat == wd::WC_ZWJ;
232 // if there's a category cached, grab it
233 cat = match self.cat {
234 None => wd::word_category(ch).2,
235 _ => self.cat.take().unwrap(),
236 };
237 take_cat = true;
238
239 // handle rule WB4
240 // just skip all format, extend, and zwj chars
241 // note that Start is a special case: if there's a bunch of Format | Extend
242 // characters at the beginning of a block of text, dump them out as one unit.
243 //
244 // (This is not obvious from the wording of UAX#29, but if you look at the
245 // test cases http://www.unicode.org/Public/UNIDATA/auxiliary/WordBreakTest.txt
246 // then the "correct" interpretation of WB4 becomes apparent.)
247 if state != Start {
248 match cat {
249 wd::WC_Extend | wd::WC_Format | wd::WC_ZWJ => {
250 skipped_format_extend = true;
251 continue;
252 }
253 _ => {}
254 }
255 }
256
257 // rule WB3c
258 // WB4 makes all ZWJs collapse into the previous state
259 // but you can still be in a Zwj state if you started with Zwj
260 //
261 // This means that an EP + Zwj will collapse into EP, which is wrong,
262 // since EP+EP is not a boundary but EP+ZWJ+EP is
263 //
264 // Thus, we separately keep track of whether or not the last character
265 // was a ZWJ. This is an additional bit of state tracked outside of the
266 // state enum; the state enum represents the last non-zwj state encountered.
267 // When prev_zwj is true, for the purposes of WB3c, we are in the Zwj state,
268 // however we are in the previous state for the purposes of all other rules.
269 if prev_zwj {
270 if is_emoji(ch) {
271 state = Emoji;
272 continue;
273 }
274 }
275 // Don't use `continue` in this match without updating `cat`
276 state = match state {
277 Start if cat == wd::WC_CR => {
278 idx += match self.get_next_cat(idx) {
279 Some(ncat) if ncat == wd::WC_LF => 1, // rule WB3
280 _ => 0,
281 };
282 break; // rule WB3a
283 }
284 Start => match cat {
285 wd::WC_ALetter => Letter, // rule WB5, WB6, WB9, WB13a
286 wd::WC_Hebrew_Letter => HLetter, // rule WB5, WB6, WB7a, WB7b, WB9, WB13a
287 wd::WC_Numeric => Numeric, // rule WB8, WB10, WB12, WB13a
288 wd::WC_Katakana => Katakana, // rule WB13, WB13a
289 wd::WC_ExtendNumLet => ExtendNumLet, // rule WB13a, WB13b
290 wd::WC_Regional_Indicator => Regional(RegionalState::Half), // rule WB13c
291 wd::WC_LF | wd::WC_Newline => break, // rule WB3a
292 wd::WC_ZWJ => Zwj, // rule WB3c
293 wd::WC_WSegSpace => WSegSpace, // rule WB3d
294 _ => {
295 if let Some(ncat) = self.get_next_cat(idx) {
296 // rule WB4
297 if ncat == wd::WC_Format || ncat == wd::WC_Extend || ncat == wd::WC_ZWJ
298 {
299 state = FormatExtend(AcceptNone);
300 self.cat = Some(ncat);
301 continue;
302 }
303 }
304 break; // rule WB999
305 }
306 },
307 WSegSpace => match cat {
308 wd::WC_WSegSpace if !skipped_format_extend => WSegSpace,
309 _ => {
310 take_curr = false;
311 break;
312 }
313 },
314 Zwj => {
315 // We already handle WB3c above.
316 take_curr = false;
317 break;
318 }
319 Letter | HLetter => match cat {
320 wd::WC_ALetter => Letter, // rule WB5
321 wd::WC_Hebrew_Letter => HLetter, // rule WB5
322 wd::WC_Numeric => Numeric, // rule WB9
323 wd::WC_ExtendNumLet => ExtendNumLet, // rule WB13a
324 wd::WC_Double_Quote if state == HLetter => {
325 savecat = cat;
326 saveidx = idx;
327 FormatExtend(RequireHLetter) // rule WB7b
328 }
329 wd::WC_Single_Quote if state == HLetter => {
330 FormatExtend(AcceptQLetter) // rule WB7a
331 }
332 wd::WC_MidLetter | wd::WC_MidNumLet | wd::WC_Single_Quote => {
333 savecat = cat;
334 saveidx = idx;
335 FormatExtend(RequireLetter) // rule WB6
336 }
337 _ => {
338 take_curr = false;
339 break;
340 }
341 },
342 Numeric => match cat {
343 wd::WC_Numeric => Numeric, // rule WB8
344 wd::WC_ALetter => Letter, // rule WB10
345 wd::WC_Hebrew_Letter => HLetter, // rule WB10
346 wd::WC_ExtendNumLet => ExtendNumLet, // rule WB13a
347 wd::WC_MidNum | wd::WC_MidNumLet | wd::WC_Single_Quote => {
348 savecat = cat;
349 saveidx = idx;
350 FormatExtend(RequireNumeric) // rule WB12
351 }
352 _ => {
353 take_curr = false;
354 break;
355 }
356 },
357 Katakana => match cat {
358 wd::WC_Katakana => Katakana, // rule WB13
359 wd::WC_ExtendNumLet => ExtendNumLet, // rule WB13a
360 _ => {
361 take_curr = false;
362 break;
363 }
364 },
365 ExtendNumLet => match cat {
366 wd::WC_ExtendNumLet => ExtendNumLet, // rule WB13a
367 wd::WC_ALetter => Letter, // rule WB13b
368 wd::WC_Hebrew_Letter => HLetter, // rule WB13b
369 wd::WC_Numeric => Numeric, // rule WB13b
370 wd::WC_Katakana => Katakana, // rule WB13b
371 _ => {
372 take_curr = false;
373 break;
374 }
375 },
376 Regional(RegionalState::Full) => {
377 // if it reaches here we've gone too far,
378 // a full flag can only compose with ZWJ/Extend/Format
379 // proceeding it.
380 take_curr = false;
381 break;
382 }
383 Regional(RegionalState::Half) => match cat {
384 wd::WC_Regional_Indicator => Regional(RegionalState::Full), // rule WB13c
385 _ => {
386 take_curr = false;
387 break;
388 }
389 },
390 Regional(_) => {
391 unreachable!("RegionalState::Unknown should not occur on forward iteration")
392 }
393 Emoji => {
394 // We already handle WB3c above. If you've reached this point, the emoji sequence is over.
395 take_curr = false;
396 break;
397 }
398 FormatExtend(t) => match t {
399 // handle FormatExtends depending on what type
400 RequireNumeric if cat == wd::WC_Numeric => Numeric, // rule WB11
401 RequireLetter | AcceptQLetter if cat == wd::WC_ALetter => Letter, // rule WB7
402 RequireLetter | AcceptQLetter if cat == wd::WC_Hebrew_Letter => HLetter, // WB7a
403 RequireHLetter if cat == wd::WC_Hebrew_Letter => HLetter, // rule WB7b
404 AcceptNone | AcceptQLetter => {
405 take_curr = false; // emit all the Format|Extend characters
406 take_cat = false;
407 break;
408 }
409 _ => break, // rewind (in if statement below)
410 },
411 }
412 }
413
414 if let FormatExtend(t) = state {
415 // we were looking for something and didn't find it; we have to back up
416 if t == RequireLetter || t == RequireHLetter || t == RequireNumeric {
417 idx = saveidx;
418 cat = savecat;
419 take_curr = false;
420 }
421 }
422
423 self.cat = if take_curr {
424 idx = idx + self.string[idx..].chars().next().unwrap().len_utf8();
425 None
426 } else if take_cat {
427 Some(cat)
428 } else {
429 None
430 };
431
432 let retstr = &self.string[..idx];
433 self.string = &self.string[idx..];
434 Some(retstr)
435 }
436 }
437
438 impl<'a> DoubleEndedIterator for UWordBounds<'a> {
439 #[inline]
440 fn next_back(&mut self) -> Option<&'a str> {
441 use self::FormatExtendType::*;
442 use self::UWordBoundsState::*;
443 use crate::tables::word as wd;
444 if self.string.len() == 0 {
445 return None;
446 }
447
448 let mut take_curr = true;
449 let mut take_cat = true;
450 let mut idx = self.string.len();
451 idx -= self.string.chars().next_back().unwrap().len_utf8();
452 let mut previdx = idx;
453 let mut saveidx = idx;
454 let mut state = Start;
455 let mut savestate = Start;
456 let mut cat = wd::WC_Any;
457
458 let mut skipped_format_extend = false;
459
460 for (curr, ch) in self.string.char_indices().rev() {
461 previdx = idx;
462 idx = curr;
463
464 // if there's a category cached, grab it
465 cat = match self.catb {
466 None => wd::word_category(ch).2,
467 _ => self.catb.take().unwrap(),
468 };
469 take_cat = true;
470
471 // backward iterator over word boundaries. Mostly the same as the forward
472 // iterator, with two weirdnesses:
473 // (1) If we encounter a single quote in the Start state, we have to check for a
474 // Hebrew Letter immediately before it.
475 // (2) Format and Extend char handling takes some gymnastics.
476
477 if cat == wd::WC_Extend || cat == wd::WC_Format || (cat == wd::WC_ZWJ && state != Zwj) {
478 // WB3c has more priority so we should not
479 // fold in that case
480 if match state {
481 FormatExtend(_) | Start => false,
482 _ => true,
483 } {
484 saveidx = previdx;
485 savestate = state;
486 state = FormatExtend(AcceptNone);
487 }
488
489 if state != Start {
490 continue;
491 }
492 } else if state == FormatExtend(AcceptNone) {
493 // finished a scan of some Format|Extend chars, restore previous state
494 state = savestate;
495 previdx = saveidx;
496 take_cat = false;
497 skipped_format_extend = true;
498 }
499
500 // Don't use `continue` in this match without updating `catb`
501 state = match state {
502 Start | FormatExtend(AcceptAny) => match cat {
503 _ if is_emoji(ch) => Zwj,
504 wd::WC_ALetter => Letter, // rule WB5, WB7, WB10, WB13b
505 wd::WC_Hebrew_Letter => HLetter, // rule WB5, WB7, WB7c, WB10, WB13b
506 wd::WC_Numeric => Numeric, // rule WB8, WB9, WB11, WB13b
507 wd::WC_Katakana => Katakana, // rule WB13, WB13b
508 wd::WC_ExtendNumLet => ExtendNumLet, // rule WB13a
509 wd::WC_Regional_Indicator => Regional(RegionalState::Unknown), // rule WB13c
510 // rule WB4:
511 wd::WC_Extend | wd::WC_Format | wd::WC_ZWJ => FormatExtend(AcceptAny),
512 wd::WC_Single_Quote => {
513 saveidx = idx;
514 FormatExtend(AcceptQLetter) // rule WB7a
515 }
516 wd::WC_WSegSpace => WSegSpace,
517 wd::WC_CR | wd::WC_LF | wd::WC_Newline => {
518 if state == Start {
519 if cat == wd::WC_LF {
520 idx -= match self.get_prev_cat(idx) {
521 Some(pcat) if pcat == wd::WC_CR => 1, // rule WB3
522 _ => 0,
523 };
524 }
525 } else {
526 take_curr = false;
527 }
528 break; // rule WB3a
529 }
530 _ => break, // rule WB999
531 },
532 Zwj => match cat {
533 // rule WB3c
534 wd::WC_ZWJ => FormatExtend(AcceptAny),
535 _ => {
536 take_curr = false;
537 break;
538 }
539 },
540 WSegSpace => match cat {
541 // rule WB3d
542 wd::WC_WSegSpace if !skipped_format_extend => WSegSpace,
543 _ => {
544 take_curr = false;
545 break;
546 }
547 },
548 Letter | HLetter => match cat {
549 wd::WC_ALetter => Letter, // rule WB5
550 wd::WC_Hebrew_Letter => HLetter, // rule WB5
551 wd::WC_Numeric => Numeric, // rule WB10
552 wd::WC_ExtendNumLet => ExtendNumLet, // rule WB13b
553 wd::WC_Double_Quote if state == HLetter => {
554 saveidx = previdx;
555 FormatExtend(RequireHLetter) // rule WB7c
556 }
557 wd::WC_MidLetter | wd::WC_MidNumLet | wd::WC_Single_Quote => {
558 saveidx = previdx;
559 FormatExtend(RequireLetter) // rule WB7
560 }
561 _ => {
562 take_curr = false;
563 break;
564 }
565 },
566 Numeric => match cat {
567 wd::WC_Numeric => Numeric, // rule WB8
568 wd::WC_ALetter => Letter, // rule WB9
569 wd::WC_Hebrew_Letter => HLetter, // rule WB9
570 wd::WC_ExtendNumLet => ExtendNumLet, // rule WB13b
571 wd::WC_MidNum | wd::WC_MidNumLet | wd::WC_Single_Quote => {
572 saveidx = previdx;
573 FormatExtend(RequireNumeric) // rule WB11
574 }
575 _ => {
576 take_curr = false;
577 break;
578 }
579 },
580 Katakana => match cat {
581 wd::WC_Katakana => Katakana, // rule WB13
582 wd::WC_ExtendNumLet => ExtendNumLet, // rule WB13b
583 _ => {
584 take_curr = false;
585 break;
586 }
587 },
588 ExtendNumLet => match cat {
589 wd::WC_ExtendNumLet => ExtendNumLet, // rule WB13a
590 wd::WC_ALetter => Letter, // rule WB13a
591 wd::WC_Hebrew_Letter => HLetter, // rule WB13a
592 wd::WC_Numeric => Numeric, // rule WB13a
593 wd::WC_Katakana => Katakana, // rule WB13a
594 _ => {
595 take_curr = false;
596 break;
597 }
598 },
599 Regional(mut regional_state) => match cat {
600 // rule WB13c
601 wd::WC_Regional_Indicator => {
602 if regional_state == RegionalState::Unknown {
603 let count = self.string[..previdx]
604 .chars()
605 .rev()
606 .map(|c| wd::word_category(c).2)
607 .filter(|&c| {
608 !(c == wd::WC_ZWJ || c == wd::WC_Extend || c == wd::WC_Format)
609 })
610 .take_while(|&c| c == wd::WC_Regional_Indicator)
611 .count();
612 regional_state = if count % 2 == 0 {
613 RegionalState::Full
614 } else {
615 RegionalState::Half
616 };
617 }
618 if regional_state == RegionalState::Full {
619 take_curr = false;
620 break;
621 } else {
622 Regional(RegionalState::Full)
623 }
624 }
625 _ => {
626 take_curr = false;
627 break;
628 }
629 },
630 Emoji => {
631 if is_emoji(ch) {
632 // rule WB3c
633 Zwj
634 } else {
635 take_curr = false;
636 break;
637 }
638 }
639 FormatExtend(t) => match t {
640 RequireNumeric if cat == wd::WC_Numeric => Numeric, // rule WB12
641 RequireLetter if cat == wd::WC_ALetter => Letter, // rule WB6
642 RequireLetter if cat == wd::WC_Hebrew_Letter => HLetter, // rule WB6
643 AcceptQLetter if cat == wd::WC_Hebrew_Letter => HLetter, // rule WB7a
644 RequireHLetter if cat == wd::WC_Hebrew_Letter => HLetter, // rule WB7b
645 _ => break, // backtrack will happens
646 },
647 }
648 }
649
650 if let FormatExtend(t) = state {
651 // if we required something but didn't find it, backtrack
652 if t == RequireLetter
653 || t == RequireHLetter
654 || t == RequireNumeric
655 || t == AcceptNone
656 || t == AcceptQLetter
657 {
658 previdx = saveidx;
659 take_cat = false;
660 take_curr = false;
661 }
662 }
663
664 self.catb = if take_curr {
665 None
666 } else {
667 idx = previdx;
668 if take_cat {
669 Some(cat)
670 } else {
671 None
672 }
673 };
674
675 let retstr = &self.string[idx..];
676 self.string = &self.string[..idx];
677 Some(retstr)
678 }
679 }
680
681 impl<'a> UWordBounds<'a> {
682 #[inline]
683 /// View the underlying data (the part yet to be iterated) as a slice of the original string.
684 ///
685 /// ```rust
686 /// # use unicode_segmentation::UnicodeSegmentation;
687 /// let mut iter = "Hello world".split_word_bounds();
688 /// assert_eq!(iter.as_str(), "Hello world");
689 /// iter.next();
690 /// assert_eq!(iter.as_str(), " world");
691 /// iter.next();
692 /// assert_eq!(iter.as_str(), "world");
693 /// ```
694 pub fn as_str(&self) -> &'a str {
695 self.string
696 }
697
698 #[inline]
699 fn get_next_cat(&self, idx: usize) -> Option<WordCat> {
700 use crate::tables::word as wd;
701 let nidx = idx + self.string[idx..].chars().next().unwrap().len_utf8();
702 if nidx < self.string.len() {
703 let nch = self.string[nidx..].chars().next().unwrap();
704 Some(wd::word_category(nch).2)
705 } else {
706 None
707 }
708 }
709
710 #[inline]
711 fn get_prev_cat(&self, idx: usize) -> Option<WordCat> {
712 use crate::tables::word as wd;
713 if idx > 0 {
714 let nch = self.string[..idx].chars().next_back().unwrap();
715 Some(wd::word_category(nch).2)
716 } else {
717 None
718 }
719 }
720 }
721
722 #[inline]
723 pub fn new_word_bounds<'b>(s: &'b str) -> UWordBounds<'b> {
724 UWordBounds {
725 string: s,
726 cat: None,
727 catb: None,
728 }
729 }
730
731 #[inline]
732 pub fn new_word_bound_indices<'b>(s: &'b str) -> UWordBoundIndices<'b> {
733 UWordBoundIndices {
734 start_offset: s.as_ptr() as usize,
735 iter: new_word_bounds(s),
736 }
737 }
738
739 #[inline]
740 fn has_alphanumeric(s: &&str) -> bool {
741 use crate::tables::util::is_alphanumeric;
742
743 s.chars().any(|c| is_alphanumeric(c))
744 }
745
746 #[inline]
747 pub fn new_unicode_words<'b>(s: &'b str) -> UnicodeWords<'b> {
748 use super::UnicodeSegmentation;
749
750 UnicodeWords {
751 inner: s.split_word_bounds().filter(has_alphanumeric),
752 }
753 }
754
755 #[inline]
756 pub fn new_unicode_word_indices<'b>(s: &'b str) -> UnicodeWordIndices<'b> {
757 use super::UnicodeSegmentation;
758
759 UnicodeWordIndices {
760 inner: s
761 .split_word_bound_indices()
762 .filter(|(_, c)| has_alphanumeric(c)),
763 }
764 }