Debian Codegraph @

.cargo/vendor/proc-macro2/tests/test.rs

raw
1 #![allow(
2 clippy::assertions_on_result_states,
3 clippy::items_after_statements,
4 clippy::needless_pass_by_value,
5 clippy::needless_raw_string_hashes,
6 clippy::non_ascii_literal,
7 clippy::octal_escapes,
8 clippy::uninlined_format_args
9 )]
10
11 use proc_macro2::{Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree};
12 use std::ffi::CStr;
13 use std::iter;
14 use std::str::{self, FromStr};
15
16 #[test]
17 fn idents() {
18 assert_eq!(
19 Ident::new("String", Span::call_site()).to_string(),
20 "String"
21 );
22 assert_eq!(Ident::new("fn", Span::call_site()).to_string(), "fn");
23 assert_eq!(Ident::new("_", Span::call_site()).to_string(), "_");
24 }
25
26 #[test]
27 fn raw_idents() {
28 assert_eq!(
29 Ident::new_raw("String", Span::call_site()).to_string(),
30 "r#String"
31 );
32 assert_eq!(Ident::new_raw("fn", Span::call_site()).to_string(), "r#fn");
33 }
34
35 #[test]
36 #[should_panic(expected = "`r#_` cannot be a raw identifier")]
37 fn ident_raw_underscore() {
38 Ident::new_raw("_", Span::call_site());
39 }
40
41 #[test]
42 #[should_panic(expected = "`r#super` cannot be a raw identifier")]
43 fn ident_raw_reserved() {
44 Ident::new_raw("super", Span::call_site());
45 }
46
47 #[test]
48 #[should_panic(expected = "Ident is not allowed to be empty; use Option<Ident>")]
49 fn ident_empty() {
50 Ident::new("", Span::call_site());
51 }
52
53 #[test]
54 #[should_panic(expected = "Ident cannot be a number; use Literal instead")]
55 fn ident_number() {
56 Ident::new("255", Span::call_site());
57 }
58
59 #[test]
60 #[should_panic(expected = "\"a#\" is not a valid Ident")]
61 fn ident_invalid() {
62 Ident::new("a#", Span::call_site());
63 }
64
65 #[test]
66 #[should_panic(expected = "not a valid Ident")]
67 fn raw_ident_empty() {
68 Ident::new("r#", Span::call_site());
69 }
70
71 #[test]
72 #[should_panic(expected = "not a valid Ident")]
73 fn raw_ident_number() {
74 Ident::new("r#255", Span::call_site());
75 }
76
77 #[test]
78 #[should_panic(expected = "\"r#a#\" is not a valid Ident")]
79 fn raw_ident_invalid() {
80 Ident::new("r#a#", Span::call_site());
81 }
82
83 #[test]
84 #[should_panic(expected = "not a valid Ident")]
85 fn lifetime_empty() {
86 Ident::new("'", Span::call_site());
87 }
88
89 #[test]
90 #[should_panic(expected = "not a valid Ident")]
91 fn lifetime_number() {
92 Ident::new("'255", Span::call_site());
93 }
94
95 #[test]
96 #[should_panic(expected = r#""'a#" is not a valid Ident"#)]
97 fn lifetime_invalid() {
98 Ident::new("'a#", Span::call_site());
99 }
100
101 #[test]
102 fn literal_string() {
103 #[track_caller]
104 fn assert(literal: Literal, expected: &str) {
105 assert_eq!(literal.to_string(), expected.trim());
106 }
107
108 assert(Literal::string(""), r#" "" "#);
109 assert(Literal::string("aA"), r#" "aA" "#);
110 assert(Literal::string("\t"), r#" "\t" "#);
111 assert(Literal::string("❤"), r#" "❤" "#);
112 assert(Literal::string("'"), r#" "'" "#);
113 assert(Literal::string("\""), r#" "\"" "#);
114 assert(Literal::string("\0"), r#" "\0" "#);
115 assert(Literal::string("\u{1}"), r#" "\u{1}" "#);
116 assert(
117 Literal::string("a\00b\07c\08d\0e\0"),
118 r#" "a\x000b\x007c\08d\0e\0" "#,
119 );
120
121 "\"\\\r\n x\"".parse::<TokenStream>().unwrap();
122 "\"\\\r\n \rx\"".parse::<TokenStream>().unwrap_err();
123 }
124
125 #[test]
126 fn literal_raw_string() {
127 "r\"\r\n\"".parse::<TokenStream>().unwrap();
128
129 fn raw_string_literal_with_hashes(n: usize) -> String {
130 let mut literal = String::new();
131 literal.push('r');
132 literal.extend(iter::repeat('#').take(n));
133 literal.push('"');
134 literal.push('"');
135 literal.extend(iter::repeat('#').take(n));
136 literal
137 }
138
139 raw_string_literal_with_hashes(255)
140 .parse::<TokenStream>()
141 .unwrap();
142
143 // https://github.com/rust-lang/rust/pull/95251
144 raw_string_literal_with_hashes(256)
145 .parse::<TokenStream>()
146 .unwrap_err();
147 }
148
149 #[cfg(procmacro2_semver_exempt)]
150 #[test]
151 fn literal_string_value() {
152 for string in ["", "...", "...\t...", "...\\...", "...\0...", "...\u{1}..."] {
153 assert_eq!(string, Literal::string(string).str_value().unwrap());
154 assert_eq!(
155 string,
156 format!("r\"{string}\"")
157 .parse::<Literal>()
158 .unwrap()
159 .str_value()
160 .unwrap(),
161 );
162 assert_eq!(
163 string,
164 format!("r##\"{string}\"##")
165 .parse::<Literal>()
166 .unwrap()
167 .str_value()
168 .unwrap(),
169 );
170 }
171 }
172
173 #[test]
174 fn literal_byte_character() {
175 #[track_caller]
176 fn assert(literal: Literal, expected: &str) {
177 assert_eq!(literal.to_string(), expected.trim());
178 }
179
180 assert(Literal::byte_character(b'a'), r#" b'a' "#);
181 assert(Literal::byte_character(b'\0'), r#" b'\0' "#);
182 assert(Literal::byte_character(b'\t'), r#" b'\t' "#);
183 assert(Literal::byte_character(b'\n'), r#" b'\n' "#);
184 assert(Literal::byte_character(b'\r'), r#" b'\r' "#);
185 assert(Literal::byte_character(b'\''), r#" b'\'' "#);
186 assert(Literal::byte_character(b'\\'), r#" b'\\' "#);
187 assert(Literal::byte_character(b'\x1f'), r#" b'\x1F' "#);
188 assert(Literal::byte_character(b'"'), r#" b'"' "#);
189 }
190
191 #[test]
192 fn literal_byte_string() {
193 #[track_caller]
194 fn assert(literal: Literal, expected: &str) {
195 assert_eq!(literal.to_string(), expected.trim());
196 }
197
198 assert(Literal::byte_string(b""), r#" b"" "#);
199 assert(Literal::byte_string(b"\0"), r#" b"\0" "#);
200 assert(Literal::byte_string(b"\t"), r#" b"\t" "#);
201 assert(Literal::byte_string(b"\n"), r#" b"\n" "#);
202 assert(Literal::byte_string(b"\r"), r#" b"\r" "#);
203 assert(Literal::byte_string(b"\""), r#" b"\"" "#);
204 assert(Literal::byte_string(b"\\"), r#" b"\\" "#);
205 assert(Literal::byte_string(b"\x1f"), r#" b"\x1F" "#);
206 assert(Literal::byte_string(b"'"), r#" b"'" "#);
207 assert(
208 Literal::byte_string(b"a\00b\07c\08d\0e\0"),
209 r#" b"a\x000b\x007c\08d\0e\0" "#,
210 );
211
212 "b\"\\\r\n x\"".parse::<TokenStream>().unwrap();
213 "b\"\\\r\n \rx\"".parse::<TokenStream>().unwrap_err();
214 "b\"\\\r\n \u{a0}x\"".parse::<TokenStream>().unwrap_err();
215 "br\"\u{a0}\"".parse::<TokenStream>().unwrap_err();
216 }
217
218 #[cfg(procmacro2_semver_exempt)]
219 #[test]
220 fn literal_byte_string_value() {
221 for bytestr in [
222 &b""[..],
223 b"...",
224 b"...\t...",
225 b"...\\...",
226 b"...\0...",
227 b"...\xF0...",
228 ] {
229 assert_eq!(
230 bytestr,
231 Literal::byte_string(bytestr).byte_str_value().unwrap(),
232 );
233 if let Ok(string) = str::from_utf8(bytestr) {
234 assert_eq!(
235 bytestr,
236 format!("br\"{string}\"")
237 .parse::<Literal>()
238 .unwrap()
239 .byte_str_value()
240 .unwrap(),
241 );
242 assert_eq!(
243 bytestr,
244 format!("br##\"{string}\"##")
245 .parse::<Literal>()
246 .unwrap()
247 .byte_str_value()
248 .unwrap(),
249 );
250 }
251 }
252 }
253
254 #[test]
255 fn literal_c_string() {
256 #[track_caller]
257 fn assert(literal: Literal, expected: &str) {
258 assert_eq!(literal.to_string(), expected.trim());
259 }
260
261 assert(Literal::c_string(<&CStr>::default()), r#" c"" "#);
262 assert(
263 Literal::c_string(CStr::from_bytes_with_nul(b"aA\0").unwrap()),
264 r#" c"aA" "#,
265 );
266 assert(
267 Literal::c_string(CStr::from_bytes_with_nul(b"aA\0").unwrap()),
268 r#" c"aA" "#,
269 );
270 assert(
271 Literal::c_string(CStr::from_bytes_with_nul(b"\t\0").unwrap()),
272 r#" c"\t" "#,
273 );
274 assert(
275 Literal::c_string(CStr::from_bytes_with_nul(b"\xE2\x9D\xA4\0").unwrap()),
276 r#" c"❤" "#,
277 );
278 assert(
279 Literal::c_string(CStr::from_bytes_with_nul(b"'\0").unwrap()),
280 r#" c"'" "#,
281 );
282 assert(
283 Literal::c_string(CStr::from_bytes_with_nul(b"\"\0").unwrap()),
284 r#" c"\"" "#,
285 );
286 assert(
287 Literal::c_string(CStr::from_bytes_with_nul(b"\x7F\xFF\xFE\xCC\xB3\0").unwrap()),
288 r#" c"\u{7f}\xFF\xFE\u{333}" "#,
289 );
290
291 let strings = r###"
292 c"hello\x80我叫\u{1F980}" // from the RFC
293 cr"\"
294 cr##"Hello "world"!"##
295 c"\t\n\r\"\\"
296 "###;
297
298 let mut tokens = strings.parse::<TokenStream>().unwrap().into_iter();
299
300 for expected in &[
301 r#"c"hello\x80我叫\u{1F980}""#,
302 r#"cr"\""#,
303 r###"cr##"Hello "world"!"##"###,
304 r#"c"\t\n\r\"\\""#,
305 ] {
306 match tokens.next().unwrap() {
307 TokenTree::Literal(literal) => {
308 assert_eq!(literal.to_string(), *expected);
309 }
310 unexpected => panic!("unexpected token: {:?}", unexpected),
311 }
312 }
313
314 if let Some(unexpected) = tokens.next() {
315 panic!("unexpected token: {:?}", unexpected);
316 }
317
318 for invalid in &[r#"c"\0""#, r#"c"\x00""#, r#"c"\u{0}""#, "c\"\0\""] {
319 if let Ok(unexpected) = invalid.parse::<TokenStream>() {
320 panic!("unexpected token: {:?}", unexpected);
321 }
322 }
323 }
324
325 #[cfg(procmacro2_semver_exempt)]
326 #[test]
327 fn literal_c_string_value() {
328 for cstr in [
329 c"",
330 c"...",
331 c"...\t...",
332 c"...\\...",
333 c"...\u{1}...",
334 c"...\xF0...",
335 ] {
336 assert_eq!(
337 cstr.to_bytes_with_nul(),
338 Literal::c_string(cstr).cstr_value().unwrap(),
339 );
340 if let Ok(string) = cstr.to_str() {
341 assert_eq!(
342 cstr.to_bytes_with_nul(),
343 format!("cr\"{string}\"")
344 .parse::<Literal>()
345 .unwrap()
346 .cstr_value()
347 .unwrap(),
348 );
349 assert_eq!(
350 cstr.to_bytes_with_nul(),
351 format!("cr##\"{string}\"##")
352 .parse::<Literal>()
353 .unwrap()
354 .cstr_value()
355 .unwrap(),
356 );
357 }
358 }
359 }
360
361 #[test]
362 fn literal_character() {
363 #[track_caller]
364 fn assert(literal: Literal, expected: &str) {
365 assert_eq!(literal.to_string(), expected.trim());
366 }
367
368 assert(Literal::character('a'), r#" 'a' "#);
369 assert(Literal::character('\t'), r#" '\t' "#);
370 assert(Literal::character('❤'), r#" '❤' "#);
371 assert(Literal::character('\''), r#" '\'' "#);
372 assert(Literal::character('"'), r#" '"' "#);
373 assert(Literal::character('\0'), r#" '\0' "#);
374 assert(Literal::character('\u{1}'), r#" '\u{1}' "#);
375 }
376
377 #[test]
378 fn literal_integer() {
379 #[track_caller]
380 fn assert(literal: Literal, expected: &str) {
381 assert_eq!(literal.to_string(), expected);
382 }
383
384 assert(Literal::u8_suffixed(10), "10u8");
385 assert(Literal::u16_suffixed(10), "10u16");
386 assert(Literal::u32_suffixed(10), "10u32");
387 assert(Literal::u64_suffixed(10), "10u64");
388 assert(Literal::u128_suffixed(10), "10u128");
389 assert(Literal::usize_suffixed(10), "10usize");
390
391 assert(Literal::i8_suffixed(10), "10i8");
392 assert(Literal::i16_suffixed(10), "10i16");
393 assert(Literal::i32_suffixed(10), "10i32");
394 assert(Literal::i64_suffixed(10), "10i64");
395 assert(Literal::i128_suffixed(10), "10i128");
396 assert(Literal::isize_suffixed(10), "10isize");
397
398 assert(Literal::u8_unsuffixed(10), "10");
399 assert(Literal::u16_unsuffixed(10), "10");
400 assert(Literal::u32_unsuffixed(10), "10");
401 assert(Literal::u64_unsuffixed(10), "10");
402 assert(Literal::u128_unsuffixed(10), "10");
403 assert(Literal::usize_unsuffixed(10), "10");
404
405 assert(Literal::i8_unsuffixed(10), "10");
406 assert(Literal::i16_unsuffixed(10), "10");
407 assert(Literal::i32_unsuffixed(10), "10");
408 assert(Literal::i64_unsuffixed(10), "10");
409 assert(Literal::i128_unsuffixed(10), "10");
410 assert(Literal::isize_unsuffixed(10), "10");
411
412 assert(Literal::i32_suffixed(-10), "-10i32");
413 assert(Literal::i32_unsuffixed(-10), "-10");
414 }
415
416 #[test]
417 fn literal_float() {
418 #[track_caller]
419 fn assert(literal: Literal, expected: &str) {
420 assert_eq!(literal.to_string(), expected);
421 }
422
423 assert(Literal::f32_suffixed(10.0), "10f32");
424 assert(Literal::f32_suffixed(-10.0), "-10f32");
425 assert(Literal::f64_suffixed(10.0), "10f64");
426 assert(Literal::f64_suffixed(-10.0), "-10f64");
427
428 assert(Literal::f32_unsuffixed(10.0), "10.0");
429 assert(Literal::f32_unsuffixed(-10.0), "-10.0");
430 assert(Literal::f64_unsuffixed(10.0), "10.0");
431 assert(Literal::f64_unsuffixed(-10.0), "-10.0");
432
433 assert(
434 Literal::f64_unsuffixed(1e100),
435 "10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.0",
436 );
437 }
438
439 #[test]
440 fn literal_suffix() {
441 fn token_count(p: &str) -> usize {
442 p.parse::<TokenStream>().unwrap().into_iter().count()
443 }
444
445 assert_eq!(token_count("999u256"), 1);
446 assert_eq!(token_count("999r#u256"), 3);
447 assert_eq!(token_count("1."), 1);
448 assert_eq!(token_count("1.f32"), 3);
449 assert_eq!(token_count("1.0_0"), 1);
450 assert_eq!(token_count("1._0"), 3);
451 assert_eq!(token_count("1._m"), 3);
452 assert_eq!(token_count("\"\"s"), 1);
453 assert_eq!(token_count("r\"\"r"), 1);
454 assert_eq!(token_count("r#\"\"#r"), 1);
455 assert_eq!(token_count("b\"\"b"), 1);
456 assert_eq!(token_count("br\"\"br"), 1);
457 assert_eq!(token_count("br#\"\"#br"), 1);
458 assert_eq!(token_count("c\"\"c"), 1);
459 assert_eq!(token_count("cr\"\"cr"), 1);
460 assert_eq!(token_count("cr#\"\"#cr"), 1);
461 assert_eq!(token_count("'c'c"), 1);
462 assert_eq!(token_count("b'b'b"), 1);
463 assert_eq!(token_count("0E"), 1);
464 assert_eq!(token_count("0o0A"), 1);
465 assert_eq!(token_count("0E--0"), 4);
466 assert_eq!(token_count("0.0ECMA"), 1);
467 }
468
469 #[test]
470 fn literal_iter_negative() {
471 let negative_literal = Literal::i32_suffixed(-3);
472 let tokens = TokenStream::from(TokenTree::Literal(negative_literal));
473 let mut iter = tokens.into_iter();
474 match iter.next().unwrap() {
475 TokenTree::Punct(punct) => {
476 assert_eq!(punct.as_char(), '-');
477 assert_eq!(punct.spacing(), Spacing::Alone);
478 }
479 unexpected => panic!("unexpected token {:?}", unexpected),
480 }
481 match iter.next().unwrap() {
482 TokenTree::Literal(literal) => {
483 assert_eq!(literal.to_string(), "3i32");
484 }
485 unexpected => panic!("unexpected token {:?}", unexpected),
486 }
487 assert!(iter.next().is_none());
488 }
489
490 #[test]
491 fn literal_parse() {
492 assert!("1".parse::<Literal>().is_ok());
493 assert!("-1".parse::<Literal>().is_ok());
494 assert!("-1u12".parse::<Literal>().is_ok());
495 assert!("1.0".parse::<Literal>().is_ok());
496 assert!("-1.0".parse::<Literal>().is_ok());
497 assert!("-1.0f12".parse::<Literal>().is_ok());
498 assert!("'a'".parse::<Literal>().is_ok());
499 assert!("\"\n\"".parse::<Literal>().is_ok());
500 assert!("0 1".parse::<Literal>().is_err());
501 assert!(" 0".parse::<Literal>().is_err());
502 assert!("0 ".parse::<Literal>().is_err());
503 assert!("/* comment */0".parse::<Literal>().is_err());
504 assert!("0/* comment */".parse::<Literal>().is_err());
505 assert!("0// comment".parse::<Literal>().is_err());
506 assert!("- 1".parse::<Literal>().is_err());
507 assert!("- 1.0".parse::<Literal>().is_err());
508 assert!("-\"\"".parse::<Literal>().is_err());
509 }
510
511 #[test]
512 fn literal_span() {
513 let positive = "0.1".parse::<Literal>().unwrap();
514 let negative = "-0.1".parse::<Literal>().unwrap();
515 let subspan = positive.subspan(1..2);
516
517 #[cfg(not(span_locations))]
518 {
519 let _ = negative;
520 assert!(subspan.is_none());
521 }
522
523 #[cfg(span_locations)]
524 {
525 assert_eq!(positive.span().start().column, 0);
526 assert_eq!(positive.span().end().column, 3);
527 assert_eq!(negative.span().start().column, 0);
528 assert_eq!(negative.span().end().column, 4);
529 assert_eq!(subspan.unwrap().source_text().unwrap(), ".");
530 }
531
532 assert!(positive.subspan(1..4).is_none());
533 }
534
535 #[cfg(span_locations)]
536 #[test]
537 fn source_text() {
538 let input = " 𓀕 a z ";
539 let mut tokens = input
540 .parse::<proc_macro2::TokenStream>()
541 .unwrap()
542 .into_iter();
543
544 let first = tokens.next().unwrap();
545 assert_eq!("𓀕", first.span().source_text().unwrap());
546
547 let second = tokens.next().unwrap();
548 let third = tokens.next().unwrap();
549 assert_eq!("z", third.span().source_text().unwrap());
550 assert_eq!("a", second.span().source_text().unwrap());
551 }
552
553 #[test]
554 fn lifetimes() {
555 let mut tokens = "'a 'static 'struct 'r#gen 'r#prefix#lifetime"
556 .parse::<TokenStream>()
557 .unwrap()
558 .into_iter();
559 assert!(match tokens.next() {
560 Some(TokenTree::Punct(punct)) => {
561 punct.as_char() == '\'' && punct.spacing() == Spacing::Joint
562 }
563 _ => false,
564 });
565 assert!(match tokens.next() {
566 Some(TokenTree::Ident(ident)) => ident == "a",
567 _ => false,
568 });
569 assert!(match tokens.next() {
570 Some(TokenTree::Punct(punct)) => {
571 punct.as_char() == '\'' && punct.spacing() == Spacing::Joint
572 }
573 _ => false,
574 });
575 assert!(match tokens.next() {
576 Some(TokenTree::Ident(ident)) => ident == "static",
577 _ => false,
578 });
579 assert!(match tokens.next() {
580 Some(TokenTree::Punct(punct)) => {
581 punct.as_char() == '\'' && punct.spacing() == Spacing::Joint
582 }
583 _ => false,
584 });
585 assert!(match tokens.next() {
586 Some(TokenTree::Ident(ident)) => ident == "struct",
587 _ => false,
588 });
589 assert!(match tokens.next() {
590 Some(TokenTree::Punct(punct)) => {
591 punct.as_char() == '\'' && punct.spacing() == Spacing::Joint
592 }
593 _ => false,
594 });
595 assert!(match tokens.next() {
596 Some(TokenTree::Ident(ident)) => ident == "r#gen",
597 _ => false,
598 });
599 assert!(match tokens.next() {
600 Some(TokenTree::Punct(punct)) => {
601 punct.as_char() == '\'' && punct.spacing() == Spacing::Joint
602 }
603 _ => false,
604 });
605 assert!(match tokens.next() {
606 Some(TokenTree::Ident(ident)) => ident == "r#prefix",
607 _ => false,
608 });
609 assert!(match tokens.next() {
610 Some(TokenTree::Punct(punct)) => {
611 punct.as_char() == '#' && punct.spacing() == Spacing::Alone
612 }
613 _ => false,
614 });
615 assert!(match tokens.next() {
616 Some(TokenTree::Ident(ident)) => ident == "lifetime",
617 _ => false,
618 });
619
620 "' a".parse::<TokenStream>().unwrap_err();
621 "' r#gen".parse::<TokenStream>().unwrap_err();
622 "' prefix#lifetime".parse::<TokenStream>().unwrap_err();
623 "'prefix#lifetime".parse::<TokenStream>().unwrap_err();
624 "'aa'bb".parse::<TokenStream>().unwrap_err();
625 "'r#gen'a".parse::<TokenStream>().unwrap_err();
626 }
627
628 #[test]
629 fn roundtrip() {
630 fn roundtrip(p: &str) {
631 println!("parse: {}", p);
632 let s = p.parse::<TokenStream>().unwrap().to_string();
633 println!("first: {}", s);
634 let s2 = s.parse::<TokenStream>().unwrap().to_string();
635 assert_eq!(s, s2);
636 }
637 roundtrip("a");
638 roundtrip("<<");
639 roundtrip("<<=");
640 roundtrip(
641 "
642 1
643 1.0
644 1f32
645 2f64
646 1usize
647 4isize
648 4e10
649 1_000
650 1_0i32
651 8u8
652 9
653 0
654 0xffffffffffffffffffffffffffffffff
655 1x
656 1u80
657 1f320
658 ",
659 );
660 roundtrip("'a");
661 roundtrip("'_");
662 roundtrip("'static");
663 roundtrip(r"'\u{10__FFFF}'");
664 roundtrip("\"\\u{10_F0FF__}foo\\u{1_0_0_0__}\"");
665 }
666
667 #[test]
668 fn fail() {
669 fn fail(p: &str) {
670 if let Ok(s) = p.parse::<TokenStream>() {
671 panic!("should have failed to parse: {}\n{:#?}", p, s);
672 }
673 }
674 fail("' static");
675 fail("r#1");
676 fail("r#_");
677 fail("\"\\u{0000000}\""); // overlong unicode escape (rust allows at most 6 hex digits)
678 fail("\"\\u{999999}\""); // outside of valid range of char
679 fail("\"\\u{_0}\""); // leading underscore
680 fail("\"\\u{}\""); // empty
681 fail("b\"\r\""); // bare carriage return in byte string
682 fail("r\"\r\""); // bare carriage return in raw string
683 fail("\"\\\r \""); // backslash carriage return
684 fail("'aa'aa");
685 fail("br##\"\"#");
686 fail("cr##\"\"#");
687 fail("\"\\\n\u{85}\r\"");
688 }
689
690 #[cfg(span_locations)]
691 #[test]
692 fn span_test() {
693 check_spans(
694 "\
695 /// This is a document comment
696 testing 123
697 {
698 testing 234
699 }",
700 &[
701 (1, 0, 1, 30), // #
702 (1, 0, 1, 30), // [ ... ]
703 (1, 0, 1, 30), // doc
704 (1, 0, 1, 30), // =
705 (1, 0, 1, 30), // "This is..."
706 (2, 0, 2, 7), // testing
707 (2, 8, 2, 11), // 123
708 (3, 0, 5, 1), // { ... }
709 (4, 2, 4, 9), // testing
710 (4, 10, 4, 13), // 234
711 ],
712 );
713 }
714
715 #[cfg(procmacro2_semver_exempt)]
716 #[test]
717 fn default_span() {
718 let start = Span::call_site().start();
719 assert_eq!(start.line, 1);
720 assert_eq!(start.column, 0);
721 let end = Span::call_site().end();
722 assert_eq!(end.line, 1);
723 assert_eq!(end.column, 0);
724 assert_eq!(Span::call_site().file(), "<unspecified>");
725 assert!(Span::call_site().local_file().is_none());
726 }
727
728 #[cfg(procmacro2_semver_exempt)]
729 #[test]
730 fn span_join() {
731 let source1 = "aaa\nbbb"
732 .parse::<TokenStream>()
733 .unwrap()
734 .into_iter()
735 .collect::<Vec<_>>();
736 let source2 = "ccc\nddd"
737 .parse::<TokenStream>()
738 .unwrap()
739 .into_iter()
740 .collect::<Vec<_>>();
741
742 assert!(source1[0].span().file() != source2[0].span().file());
743 assert_eq!(source1[0].span().file(), source1[1].span().file());
744
745 let joined1 = source1[0].span().join(source1[1].span());
746 let joined2 = source1[0].span().join(source2[0].span());
747 assert!(joined1.is_some());
748 assert!(joined2.is_none());
749
750 let start = joined1.unwrap().start();
751 let end = joined1.unwrap().end();
752 assert_eq!(start.line, 1);
753 assert_eq!(start.column, 0);
754 assert_eq!(end.line, 2);
755 assert_eq!(end.column, 3);
756
757 assert_eq!(joined1.unwrap().file(), source1[0].span().file());
758 }
759
760 #[test]
761 fn no_panic() {
762 let s = str::from_utf8(b"b\'\xc2\x86 \x00\x00\x00^\"").unwrap();
763 assert!(s.parse::<TokenStream>().is_err());
764 }
765
766 #[test]
767 fn punct_before_comment() {
768 let mut tts = TokenStream::from_str("~// comment").unwrap().into_iter();
769 match tts.next().unwrap() {
770 TokenTree::Punct(tt) => {
771 assert_eq!(tt.as_char(), '~');
772 assert_eq!(tt.spacing(), Spacing::Alone);
773 }
774 wrong => panic!("wrong token {:?}", wrong),
775 }
776 }
777
778 #[test]
779 fn joint_last_token() {
780 // This test verifies that we match the behavior of libproc_macro *not* in
781 // the range nightly-2020-09-06 through nightly-2020-09-10, in which this
782 // behavior was temporarily broken.
783 // See https://github.com/rust-lang/rust/issues/76399
784
785 let joint_punct = Punct::new(':', Spacing::Joint);
786 let stream = TokenStream::from(TokenTree::Punct(joint_punct));
787 let TokenTree::Punct(punct) = stream.into_iter().next().unwrap() else {
788 unreachable!();
789 };
790 assert_eq!(punct.spacing(), Spacing::Joint);
791 }
792
793 #[test]
794 fn raw_identifier() {
795 let mut tts = TokenStream::from_str("r#dyn").unwrap().into_iter();
796 match tts.next().unwrap() {
797 TokenTree::Ident(raw) => assert_eq!("r#dyn", raw.to_string()),
798 wrong => panic!("wrong token {:?}", wrong),
799 }
800 assert!(tts.next().is_none());
801 }
802
803 #[test]
804 fn test_display_ident() {
805 let ident = Ident::new("proc_macro", Span::call_site());
806 assert_eq!(format!("{ident}"), "proc_macro");
807 assert_eq!(format!("{ident:-^14}"), "proc_macro");
808
809 let ident = Ident::new_raw("proc_macro", Span::call_site());
810 assert_eq!(format!("{ident}"), "r#proc_macro");
811 assert_eq!(format!("{ident:-^14}"), "r#proc_macro");
812 }
813
814 #[test]
815 fn test_debug_ident() {
816 let ident = Ident::new("proc_macro", Span::call_site());
817 let expected = if cfg!(span_locations) {
818 "Ident { sym: proc_macro }"
819 } else {
820 "Ident(proc_macro)"
821 };
822 assert_eq!(expected, format!("{:?}", ident));
823
824 let ident = Ident::new_raw("proc_macro", Span::call_site());
825 let expected = if cfg!(span_locations) {
826 "Ident { sym: r#proc_macro }"
827 } else {
828 "Ident(r#proc_macro)"
829 };
830 assert_eq!(expected, format!("{:?}", ident));
831 }
832
833 #[test]
834 fn test_display_tokenstream() {
835 let tts = TokenStream::from_str("[a + 1]").unwrap();
836 assert_eq!(format!("{tts}"), "[a + 1]");
837 assert_eq!(format!("{tts:-^5}"), "[a + 1]");
838 }
839
840 #[test]
841 fn test_debug_tokenstream() {
842 let tts = TokenStream::from_str("[a + 1]").unwrap();
843
844 #[cfg(not(span_locations))]
845 let expected = "\
846 TokenStream [
847 Group {
848 delimiter: Bracket,
849 stream: TokenStream [
850 Ident {
851 sym: a,
852 },
853 Punct {
854 char: '+',
855 spacing: Alone,
856 },
857 Literal {
858 lit: 1,
859 },
860 ],
861 },
862 ]\
863 ";
864
865 #[cfg(not(span_locations))]
866 let expected_before_trailing_commas = "\
867 TokenStream [
868 Group {
869 delimiter: Bracket,
870 stream: TokenStream [
871 Ident {
872 sym: a
873 },
874 Punct {
875 char: '+',
876 spacing: Alone
877 },
878 Literal {
879 lit: 1
880 }
881 ]
882 }
883 ]\
884 ";
885
886 #[cfg(span_locations)]
887 let expected = "\
888 TokenStream [
889 Group {
890 delimiter: Bracket,
891 stream: TokenStream [
892 Ident {
893 sym: a,
894 span: bytes(2..3),
895 },
896 Punct {
897 char: '+',
898 spacing: Alone,
899 span: bytes(4..5),
900 },
901 Literal {
902 lit: 1,
903 span: bytes(6..7),
904 },
905 ],
906 span: bytes(1..8),
907 },
908 ]\
909 ";
910
911 #[cfg(span_locations)]
912 let expected_before_trailing_commas = "\
913 TokenStream [
914 Group {
915 delimiter: Bracket,
916 stream: TokenStream [
917 Ident {
918 sym: a,
919 span: bytes(2..3)
920 },
921 Punct {
922 char: '+',
923 spacing: Alone,
924 span: bytes(4..5)
925 },
926 Literal {
927 lit: 1,
928 span: bytes(6..7)
929 }
930 ],
931 span: bytes(1..8)
932 }
933 ]\
934 ";
935
936 let actual = format!("{:#?}", tts);
937 if actual.ends_with(",\n]") {
938 assert_eq!(expected, actual);
939 } else {
940 assert_eq!(expected_before_trailing_commas, actual);
941 }
942 }
943
944 #[test]
945 fn default_tokenstream_is_empty() {
946 let default_token_stream = <TokenStream as Default>::default();
947
948 assert!(default_token_stream.is_empty());
949 }
950
951 #[test]
952 fn tokenstream_size_hint() {
953 let tokens = "a b (c d) e".parse::<TokenStream>().unwrap();
954
955 assert_eq!(tokens.into_iter().size_hint(), (4, Some(4)));
956 }
957
958 #[test]
959 fn tuple_indexing() {
960 // This behavior may change depending on https://github.com/rust-lang/rust/pull/71322
961 let mut tokens = "tuple.0.0".parse::<TokenStream>().unwrap().into_iter();
962 assert_eq!("tuple", tokens.next().unwrap().to_string());
963 assert_eq!(".", tokens.next().unwrap().to_string());
964 assert_eq!("0.0", tokens.next().unwrap().to_string());
965 assert!(tokens.next().is_none());
966 }
967
968 #[cfg(span_locations)]
969 #[test]
970 fn non_ascii_tokens() {
971 check_spans("// abc", &[]);
972 check_spans("// ábc", &[]);
973 check_spans("// abc x", &[]);
974 check_spans("// ábc x", &[]);
975 check_spans("/* abc */ x", &[(1, 10, 1, 11)]);
976 check_spans("/* ábc */ x", &[(1, 10, 1, 11)]);
977 check_spans("/* ab\nc */ x", &[(2, 5, 2, 6)]);
978 check_spans("/* áb\nc */ x", &[(2, 5, 2, 6)]);
979 check_spans("/*** abc */ x", &[(1, 12, 1, 13)]);
980 check_spans("/*** ábc */ x", &[(1, 12, 1, 13)]);
981 check_spans(r#""abc""#, &[(1, 0, 1, 5)]);
982 check_spans(r#""ábc""#, &[(1, 0, 1, 5)]);
983 check_spans(r##"r#"abc"#"##, &[(1, 0, 1, 8)]);
984 check_spans(r##"r#"ábc"#"##, &[(1, 0, 1, 8)]);
985 check_spans("r#\"a\nc\"#", &[(1, 0, 2, 3)]);
986 check_spans("r#\"á\nc\"#", &[(1, 0, 2, 3)]);
987 check_spans("'a'", &[(1, 0, 1, 3)]);
988 check_spans("'á'", &[(1, 0, 1, 3)]);
989 check_spans("//! abc", &[(1, 0, 1, 7), (1, 0, 1, 7), (1, 0, 1, 7)]);
990 check_spans("//! ábc", &[(1, 0, 1, 7), (1, 0, 1, 7), (1, 0, 1, 7)]);
991 check_spans("//! abc\n", &[(1, 0, 1, 7), (1, 0, 1, 7), (1, 0, 1, 7)]);
992 check_spans("//! ábc\n", &[(1, 0, 1, 7), (1, 0, 1, 7), (1, 0, 1, 7)]);
993 check_spans("/*! abc */", &[(1, 0, 1, 10), (1, 0, 1, 10), (1, 0, 1, 10)]);
994 check_spans("/*! ábc */", &[(1, 0, 1, 10), (1, 0, 1, 10), (1, 0, 1, 10)]);
995 check_spans("/*! a\nc */", &[(1, 0, 2, 4), (1, 0, 2, 4), (1, 0, 2, 4)]);
996 check_spans("/*! á\nc */", &[(1, 0, 2, 4), (1, 0, 2, 4), (1, 0, 2, 4)]);
997 check_spans("abc", &[(1, 0, 1, 3)]);
998 check_spans("ábc", &[(1, 0, 1, 3)]);
999 check_spans("ábć", &[(1, 0, 1, 3)]);
1000 check_spans("abc// foo", &[(1, 0, 1, 3)]);
1001 check_spans("ábc// foo", &[(1, 0, 1, 3)]);
1002 check_spans("ábć// foo", &[(1, 0, 1, 3)]);
1003 check_spans("b\"a\\\n c\"", &[(1, 0, 2, 3)]);
1004 }
1005
1006 #[cfg(span_locations)]
1007 fn check_spans(p: &str, mut lines: &[(usize, usize, usize, usize)]) {
1008 let ts = p.parse::<TokenStream>().unwrap();
1009 check_spans_internal(ts, &mut lines);
1010 assert!(lines.is_empty(), "leftover ranges: {:?}", lines);
1011 }
1012
1013 #[cfg(span_locations)]
1014 fn check_spans_internal(ts: TokenStream, lines: &mut &[(usize, usize, usize, usize)]) {
1015 for i in ts {
1016 if let Some((&(sline, scol, eline, ecol), rest)) = lines.split_first() {
1017 *lines = rest;
1018
1019 let start = i.span().start();
1020 assert_eq!(start.line, sline, "sline did not match for {}", i);
1021 assert_eq!(start.column, scol, "scol did not match for {}", i);
1022
1023 let end = i.span().end();
1024 assert_eq!(end.line, eline, "eline did not match for {}", i);
1025 assert_eq!(end.column, ecol, "ecol did not match for {}", i);
1026
1027 if let TokenTree::Group(g) = i {
1028 check_spans_internal(g.stream().clone(), lines);
1029 }
1030 }
1031 }
1032 }
1033
1034 #[test]
1035 fn whitespace() {
1036 // space, horizontal tab, vertical tab, form feed, carriage return, line
1037 // feed, non-breaking space, left-to-right mark, right-to-left mark
1038 let various_spaces = " \t\u{b}\u{c}\r\n\u{a0}\u{200e}\u{200f}";
1039 let tokens = various_spaces.parse::<TokenStream>().unwrap();
1040 assert_eq!(tokens.into_iter().count(), 0);
1041
1042 let lone_carriage_returns = " \r \r\r\n ";
1043 lone_carriage_returns.parse::<TokenStream>().unwrap();
1044 }
1045
1046 #[test]
1047 fn byte_order_mark() {
1048 let string = "\u{feff}foo";
1049 let tokens = string.parse::<TokenStream>().unwrap();
1050 match tokens.into_iter().next().unwrap() {
1051 TokenTree::Ident(ident) => assert_eq!(ident, "foo"),
1052 _ => unreachable!(),
1053 }
1054
1055 let string = "foo\u{feff}";
1056 string.parse::<TokenStream>().unwrap_err();
1057 }
1058
1059 #[cfg(span_locations)]
1060 fn create_span() -> proc_macro2::Span {
1061 let tts: TokenStream = "1".parse().unwrap();
1062 match tts.into_iter().next().unwrap() {
1063 TokenTree::Literal(literal) => literal.span(),
1064 _ => unreachable!(),
1065 }
1066 }
1067
1068 #[cfg(span_locations)]
1069 #[test]
1070 fn test_invalidate_current_thread_spans() {
1071 let actual = format!("{:#?}", create_span());
1072 assert_eq!(actual, "bytes(1..2)");
1073 let actual = format!("{:#?}", create_span());
1074 assert_eq!(actual, "bytes(3..4)");
1075
1076 proc_macro2::extra::invalidate_current_thread_spans();
1077
1078 let actual = format!("{:#?}", create_span());
1079 // Test that span offsets have been reset after the call
1080 // to invalidate_current_thread_spans()
1081 assert_eq!(actual, "bytes(1..2)");
1082 }
1083
1084 #[cfg(span_locations)]
1085 #[test]
1086 #[should_panic(expected = "Invalid span with no related FileInfo!")]
1087 fn test_use_span_after_invalidation() {
1088 let span = create_span();
1089
1090 proc_macro2::extra::invalidate_current_thread_spans();
1091
1092 span.source_text();
1093 }