| 1 |
//! Tokens representing Rust punctuation, keywords, and delimiters. |
| 2 |
//! |
| 3 |
//! The type names in this module can be difficult to keep straight, so we |
| 4 |
//! prefer to use the [`Token!`] macro instead. This is a type-macro that |
| 5 |
//! expands to the token type of the given token. |
| 6 |
//! |
| 7 |
//! [`Token!`]: crate::Token |
| 8 |
//! |
| 9 |
//! # Example |
| 10 |
//! |
| 11 |
//! The [`ItemStatic`] syntax tree node is defined like this. |
| 12 |
//! |
| 13 |
//! [`ItemStatic`]: crate::ItemStatic |
| 14 |
//! |
| 15 |
//! ``` |
| 16 |
//! # use syn::{Attribute, Expr, Ident, Token, Type, Visibility}; |
| 17 |
//! # |
| 18 |
//! pub struct ItemStatic { |
| 19 |
//! pub attrs: Vec<Attribute>, |
| 20 |
//! pub vis: Visibility, |
| 21 |
//! pub static_token: Token![static], |
| 22 |
//! pub mutability: Option<Token![mut]>, |
| 23 |
//! pub ident: Ident, |
| 24 |
//! pub colon_token: Token![:], |
| 25 |
//! pub ty: Box<Type>, |
| 26 |
//! pub eq_token: Token![=], |
| 27 |
//! pub expr: Box<Expr>, |
| 28 |
//! pub semi_token: Token![;], |
| 29 |
//! } |
| 30 |
//! ``` |
| 31 |
//! |
| 32 |
//! # Parsing |
| 33 |
//! |
| 34 |
//! Keywords and punctuation can be parsed through the [`ParseStream::parse`] |
| 35 |
//! method. Delimiter tokens are parsed using the [`parenthesized!`], |
| 36 |
//! [`bracketed!`] and [`braced!`] macros. |
| 37 |
//! |
| 38 |
//! [`ParseStream::parse`]: crate::parse::ParseBuffer::parse() |
| 39 |
//! [`parenthesized!`]: crate::parenthesized! |
| 40 |
//! [`bracketed!`]: crate::bracketed! |
| 41 |
//! [`braced!`]: crate::braced! |
| 42 |
//! |
| 43 |
//! ``` |
| 44 |
//! use syn::{Attribute, Result}; |
| 45 |
//! use syn::parse::{Parse, ParseStream}; |
| 46 |
//! # |
| 47 |
//! # enum ItemStatic {} |
| 48 |
//! |
| 49 |
//! // Parse the ItemStatic struct shown above. |
| 50 |
//! impl Parse for ItemStatic { |
| 51 |
//! fn parse(input: ParseStream) -> Result<Self> { |
| 52 |
//! # use syn::ItemStatic; |
| 53 |
//! # fn parse(input: ParseStream) -> Result<ItemStatic> { |
| 54 |
//! Ok(ItemStatic { |
| 55 |
//! attrs: input.call(Attribute::parse_outer)?, |
| 56 |
//! vis: input.parse()?, |
| 57 |
//! static_token: input.parse()?, |
| 58 |
//! mutability: input.parse()?, |
| 59 |
//! ident: input.parse()?, |
| 60 |
//! colon_token: input.parse()?, |
| 61 |
//! ty: input.parse()?, |
| 62 |
//! eq_token: input.parse()?, |
| 63 |
//! expr: input.parse()?, |
| 64 |
//! semi_token: input.parse()?, |
| 65 |
//! }) |
| 66 |
//! # } |
| 67 |
//! # unimplemented!() |
| 68 |
//! } |
| 69 |
//! } |
| 70 |
//! ``` |
| 71 |
//! |
| 72 |
//! # Other operations |
| 73 |
//! |
| 74 |
//! Every keyword and punctuation token supports the following operations. |
| 75 |
//! |
| 76 |
//! - [Peeking] — `input.peek(Token![...])` |
| 77 |
//! |
| 78 |
//! - [Parsing] — `input.parse::<Token![...]>()?` |
| 79 |
//! |
| 80 |
//! - [Printing] — `quote!( ... #the_token ... )` |
| 81 |
//! |
| 82 |
//! - Construction from a [`Span`] — `let the_token = Token` |
| 83 |
//! |
| 84 |
//! - Field access to its span — `let sp = the_token.span` |
| 85 |
//! |
| 86 |
//! [Peeking]: crate::parse::ParseBuffer::peek() |
| 87 |
//! [Parsing]: crate::parse::ParseBuffer::parse() |
| 88 |
//! [Printing]: https://docs.rs/quote/1.0/quote/trait.ToTokens.html |
| 89 |
//! [`Span`]: https://docs.rs/proc-macro2/1.0/proc_macro2/struct.Span.html |
| 90 |
|
| 91 |
use self::private::WithSpan; |
| 92 |
#[cfg(feature = "parsing")] |
| 93 |
use crate::buffer::Cursor; |
| 94 |
#[cfg(feature = "parsing")] |
| 95 |
use crate::error::Result; |
| 96 |
#[cfg(feature = "parsing")] |
| 97 |
use crate::lifetime::Lifetime; |
| 98 |
#[cfg(feature = "parsing")] |
| 99 |
use crate::lit::{Lit, LitBool, LitByte, LitByteStr, LitChar, LitFloat, LitInt, LitStr}; |
| 100 |
#[cfg(feature = "parsing")] |
| 101 |
use crate::lookahead; |
| 102 |
#[cfg(feature = "parsing")] |
| 103 |
use crate::parse::{Parse, ParseStream}; |
| 104 |
use crate::span::IntoSpans; |
| 105 |
#[cfg(any(feature = "parsing", feature = "printing"))] |
| 106 |
use proc_macro2::Ident; |
| 107 |
use proc_macro2::Span; |
| 108 |
#[cfg(feature = "printing")] |
| 109 |
use proc_macro2::TokenStream; |
| 110 |
#[cfg(feature = "parsing")] |
| 111 |
use proc_macro2::{Delimiter, Literal, Punct, TokenTree}; |
| 112 |
#[cfg(feature = "printing")] |
| 113 |
use quote::{ToTokens, TokenStreamExt}; |
| 114 |
#[cfg(feature = "extra-traits")] |
| 115 |
use std::cmp; |
| 116 |
#[cfg(feature = "extra-traits")] |
| 117 |
use std::fmt::{self, Debug}; |
| 118 |
#[cfg(feature = "extra-traits")] |
| 119 |
use std::hash::{Hash, Hasher}; |
| 120 |
use std::ops::{Deref, DerefMut}; |
| 121 |
|
| 122 |
/// Marker trait for types that represent single tokens. |
| 123 |
/// |
| 124 |
/// This trait is sealed and cannot be implemented for types outside of Syn. |
| 125 |
#[cfg(feature = "parsing")] |
| 126 |
pub trait Token: private::Sealed { |
| 127 |
// Not public API. |
| 128 |
#[doc(hidden)] |
| 129 |
fn peek(cursor: Cursor) -> bool; |
| 130 |
|
| 131 |
// Not public API. |
| 132 |
#[doc(hidden)] |
| 133 |
fn display() -> &'static str; |
| 134 |
} |
| 135 |
|
| 136 |
mod private { |
| 137 |
use proc_macro2::Span; |
| 138 |
|
| 139 |
#[cfg(feature = "parsing")] |
| 140 |
pub trait Sealed {} |
| 141 |
|
| 142 |
/// Support writing `token.span` rather than `token.spans[0]` on tokens that |
| 143 |
/// hold a single span. |
| 144 |
#[repr(C)] |
| 145 |
pub struct WithSpan { |
| 146 |
pub span: Span, |
| 147 |
} |
| 148 |
} |
| 149 |
|
| 150 |
#[cfg(feature = "parsing")] |
| 151 |
impl private::Sealed for Ident {} |
| 152 |
|
| 153 |
#[cfg(feature = "parsing")] |
| 154 |
fn peek_impl(cursor: Cursor, peek: fn(ParseStream) -> bool) -> bool { |
| 155 |
use crate::parse::Unexpected; |
| 156 |
use std::cell::Cell; |
| 157 |
use std::rc::Rc; |
| 158 |
|
| 159 |
let scope = Span::call_site(); |
| 160 |
let unexpected = Rc::new(Cell::new(Unexpected::None)); |
| 161 |
let buffer = crate::parse::new_parse_buffer(scope, cursor, unexpected); |
| 162 |
peek(&buffer) |
| 163 |
} |
| 164 |
|
| 165 |
macro_rules! impl_token { |
| 166 |
($display:tt $name:ty) => { |
| 167 |
#[cfg(feature = "parsing")] |
| 168 |
impl Token for $name { |
| 169 |
fn peek(cursor: Cursor) -> bool { |
| 170 |
fn peek(input: ParseStream) -> bool { |
| 171 |
<$name as Parse>::parse(input).is_ok() |
| 172 |
} |
| 173 |
peek_impl(cursor, peek) |
| 174 |
} |
| 175 |
|
| 176 |
fn display() -> &'static str { |
| 177 |
$display |
| 178 |
} |
| 179 |
} |
| 180 |
|
| 181 |
#[cfg(feature = "parsing")] |
| 182 |
impl private::Sealed for $name {} |
| 183 |
}; |
| 184 |
} |
| 185 |
|
| 186 |
impl_token!("lifetime" Lifetime); |
| 187 |
impl_token!("literal" Lit); |
| 188 |
impl_token!("string literal" LitStr); |
| 189 |
impl_token!("byte string literal" LitByteStr); |
| 190 |
impl_token!("byte literal" LitByte); |
| 191 |
impl_token!("character literal" LitChar); |
| 192 |
impl_token!("integer literal" LitInt); |
| 193 |
impl_token!("floating point literal" LitFloat); |
| 194 |
impl_token!("boolean literal" LitBool); |
| 195 |
impl_token!("group token" proc_macro2::Group); |
| 196 |
|
| 197 |
macro_rules! impl_low_level_token { |
| 198 |
($display:tt $ty:ident $get:ident) => { |
| 199 |
#[cfg(feature = "parsing")] |
| 200 |
impl Token for $ty { |
| 201 |
fn peek(cursor: Cursor) -> bool { |
| 202 |
cursor.$get().is_some() |
| 203 |
} |
| 204 |
|
| 205 |
fn display() -> &'static str { |
| 206 |
$display |
| 207 |
} |
| 208 |
} |
| 209 |
|
| 210 |
#[cfg(feature = "parsing")] |
| 211 |
impl private::Sealed for $ty {} |
| 212 |
}; |
| 213 |
} |
| 214 |
|
| 215 |
impl_low_level_token!("punctuation token" Punct punct); |
| 216 |
impl_low_level_token!("literal" Literal literal); |
| 217 |
impl_low_level_token!("token" TokenTree token_tree); |
| 218 |
|
| 219 |
// Not public API. |
| 220 |
#[doc(hidden)] |
| 221 |
#[cfg(feature = "parsing")] |
| 222 |
pub trait CustomToken { |
| 223 |
fn peek(cursor: Cursor) -> bool; |
| 224 |
fn display() -> &'static str; |
| 225 |
} |
| 226 |
|
| 227 |
#[cfg(feature = "parsing")] |
| 228 |
impl<T: CustomToken> private::Sealed for T {} |
| 229 |
|
| 230 |
#[cfg(feature = "parsing")] |
| 231 |
impl<T: CustomToken> Token for T { |
| 232 |
fn peek(cursor: Cursor) -> bool { |
| 233 |
<Self as CustomToken>::peek(cursor) |
| 234 |
} |
| 235 |
|
| 236 |
fn display() -> &'static str { |
| 237 |
<Self as CustomToken>::display() |
| 238 |
} |
| 239 |
} |
| 240 |
|
| 241 |
macro_rules! define_keywords { |
| 242 |
($($token:tt pub struct $name:ident #[$doc:meta])*) => { |
| 243 |
$( |
| 244 |
#[$doc] |
| 245 |
/// |
| 246 |
/// Don't try to remember the name of this type — use the |
| 247 |
/// [`Token!`] macro instead. |
| 248 |
/// |
| 249 |
/// [`Token!`]: crate::token |
| 250 |
pub struct $name { |
| 251 |
pub span: Span, |
| 252 |
} |
| 253 |
|
| 254 |
#[doc(hidden)] |
| 255 |
#[allow(non_snake_case)] |
| 256 |
pub fn $name<S: IntoSpans<[Span; 1]>>(span: S) -> $name { |
| 257 |
$name { |
| 258 |
span: span.into_spans()[0], |
| 259 |
} |
| 260 |
} |
| 261 |
|
| 262 |
impl std::default::Default for $name { |
| 263 |
fn default() -> Self { |
| 264 |
$name { |
| 265 |
span: Span::call_site(), |
| 266 |
} |
| 267 |
} |
| 268 |
} |
| 269 |
|
| 270 |
#[cfg(feature = "clone-impls")] |
| 271 |
#[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))] |
| 272 |
impl Copy for $name {} |
| 273 |
|
| 274 |
#[cfg(feature = "clone-impls")] |
| 275 |
#[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))] |
| 276 |
impl Clone for $name { |
| 277 |
fn clone(&self) -> Self { |
| 278 |
*self |
| 279 |
} |
| 280 |
} |
| 281 |
|
| 282 |
#[cfg(feature = "extra-traits")] |
| 283 |
#[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))] |
| 284 |
impl Debug for $name { |
| 285 |
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
| 286 |
f.write_str(stringify!($name)) |
| 287 |
} |
| 288 |
} |
| 289 |
|
| 290 |
#[cfg(feature = "extra-traits")] |
| 291 |
#[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))] |
| 292 |
impl cmp::Eq for $name {} |
| 293 |
|
| 294 |
#[cfg(feature = "extra-traits")] |
| 295 |
#[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))] |
| 296 |
impl PartialEq for $name { |
| 297 |
fn eq(&self, _other: &$name) -> bool { |
| 298 |
true |
| 299 |
} |
| 300 |
} |
| 301 |
|
| 302 |
#[cfg(feature = "extra-traits")] |
| 303 |
#[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))] |
| 304 |
impl Hash for $name { |
| 305 |
fn hash<H: Hasher>(&self, _state: &mut H) {} |
| 306 |
} |
| 307 |
|
| 308 |
#[cfg(feature = "printing")] |
| 309 |
#[cfg_attr(doc_cfg, doc(cfg(feature = "printing")))] |
| 310 |
impl ToTokens for $name { |
| 311 |
fn to_tokens(&self, tokens: &mut TokenStream) { |
| 312 |
printing::keyword($token, self.span, tokens); |
| 313 |
} |
| 314 |
} |
| 315 |
|
| 316 |
#[cfg(feature = "parsing")] |
| 317 |
#[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] |
| 318 |
impl Parse for $name { |
| 319 |
fn parse(input: ParseStream) -> Result<Self> { |
| 320 |
Ok($name { |
| 321 |
span: parsing::keyword(input, $token)?, |
| 322 |
}) |
| 323 |
} |
| 324 |
} |
| 325 |
|
| 326 |
#[cfg(feature = "parsing")] |
| 327 |
impl Token for $name { |
| 328 |
fn peek(cursor: Cursor) -> bool { |
| 329 |
parsing::peek_keyword(cursor, $token) |
| 330 |
} |
| 331 |
|
| 332 |
fn display() -> &'static str { |
| 333 |
concat!("`", $token, "`") |
| 334 |
} |
| 335 |
} |
| 336 |
|
| 337 |
#[cfg(feature = "parsing")] |
| 338 |
impl private::Sealed for $name {} |
| 339 |
)* |
| 340 |
}; |
| 341 |
} |
| 342 |
|
| 343 |
macro_rules! impl_deref_if_len_is_1 { |
| 344 |
($name:ident/1) => { |
| 345 |
impl Deref for $name { |
| 346 |
type Target = WithSpan; |
| 347 |
|
| 348 |
fn deref(&self) -> &Self::Target { |
| 349 |
unsafe { &*(self as *const Self as *const WithSpan) } |
| 350 |
} |
| 351 |
} |
| 352 |
|
| 353 |
impl DerefMut for $name { |
| 354 |
fn deref_mut(&mut self) -> &mut Self::Target { |
| 355 |
unsafe { &mut *(self as *mut Self as *mut WithSpan) } |
| 356 |
} |
| 357 |
} |
| 358 |
}; |
| 359 |
|
| 360 |
($name:ident/$len:tt) => {}; |
| 361 |
} |
| 362 |
|
| 363 |
macro_rules! define_punctuation_structs { |
| 364 |
($($token:tt pub struct $name:ident/$len:tt #[$doc:meta])*) => { |
| 365 |
$( |
| 366 |
#[repr(C)] |
| 367 |
#[$doc] |
| 368 |
/// |
| 369 |
/// Don't try to remember the name of this type — use the |
| 370 |
/// [`Token!`] macro instead. |
| 371 |
/// |
| 372 |
/// [`Token!`]: crate::token |
| 373 |
pub struct $name { |
| 374 |
pub spans: [Span; $len], |
| 375 |
} |
| 376 |
|
| 377 |
#[doc(hidden)] |
| 378 |
#[allow(non_snake_case)] |
| 379 |
pub fn $name<S: IntoSpans<[Span; $len]>>(spans: S) -> $name { |
| 380 |
$name { |
| 381 |
spans: spans.into_spans(), |
| 382 |
} |
| 383 |
} |
| 384 |
|
| 385 |
impl std::default::Default for $name { |
| 386 |
fn default() -> Self { |
| 387 |
$name { |
| 388 |
spans: [Span::call_site(); $len], |
| 389 |
} |
| 390 |
} |
| 391 |
} |
| 392 |
|
| 393 |
#[cfg(feature = "clone-impls")] |
| 394 |
#[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))] |
| 395 |
impl Copy for $name {} |
| 396 |
|
| 397 |
#[cfg(feature = "clone-impls")] |
| 398 |
#[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))] |
| 399 |
impl Clone for $name { |
| 400 |
fn clone(&self) -> Self { |
| 401 |
*self |
| 402 |
} |
| 403 |
} |
| 404 |
|
| 405 |
#[cfg(feature = "extra-traits")] |
| 406 |
#[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))] |
| 407 |
impl Debug for $name { |
| 408 |
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
| 409 |
f.write_str(stringify!($name)) |
| 410 |
} |
| 411 |
} |
| 412 |
|
| 413 |
#[cfg(feature = "extra-traits")] |
| 414 |
#[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))] |
| 415 |
impl cmp::Eq for $name {} |
| 416 |
|
| 417 |
#[cfg(feature = "extra-traits")] |
| 418 |
#[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))] |
| 419 |
impl PartialEq for $name { |
| 420 |
fn eq(&self, _other: &$name) -> bool { |
| 421 |
true |
| 422 |
} |
| 423 |
} |
| 424 |
|
| 425 |
#[cfg(feature = "extra-traits")] |
| 426 |
#[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))] |
| 427 |
impl Hash for $name { |
| 428 |
fn hash<H: Hasher>(&self, _state: &mut H) {} |
| 429 |
} |
| 430 |
|
| 431 |
impl_deref_if_len_is_1!($name/$len); |
| 432 |
)* |
| 433 |
}; |
| 434 |
} |
| 435 |
|
| 436 |
macro_rules! define_punctuation { |
| 437 |
($($token:tt pub struct $name:ident/$len:tt #[$doc:meta])*) => { |
| 438 |
$( |
| 439 |
define_punctuation_structs! { |
| 440 |
$token pub struct $name/$len #[$doc] |
| 441 |
} |
| 442 |
|
| 443 |
#[cfg(feature = "printing")] |
| 444 |
#[cfg_attr(doc_cfg, doc(cfg(feature = "printing")))] |
| 445 |
impl ToTokens for $name { |
| 446 |
fn to_tokens(&self, tokens: &mut TokenStream) { |
| 447 |
printing::punct($token, &self.spans, tokens); |
| 448 |
} |
| 449 |
} |
| 450 |
|
| 451 |
#[cfg(feature = "parsing")] |
| 452 |
#[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] |
| 453 |
impl Parse for $name { |
| 454 |
fn parse(input: ParseStream) -> Result<Self> { |
| 455 |
Ok($name { |
| 456 |
spans: parsing::punct(input, $token)?, |
| 457 |
}) |
| 458 |
} |
| 459 |
} |
| 460 |
|
| 461 |
#[cfg(feature = "parsing")] |
| 462 |
impl Token for $name { |
| 463 |
fn peek(cursor: Cursor) -> bool { |
| 464 |
parsing::peek_punct(cursor, $token) |
| 465 |
} |
| 466 |
|
| 467 |
fn display() -> &'static str { |
| 468 |
concat!("`", $token, "`") |
| 469 |
} |
| 470 |
} |
| 471 |
|
| 472 |
#[cfg(feature = "parsing")] |
| 473 |
impl private::Sealed for $name {} |
| 474 |
)* |
| 475 |
}; |
| 476 |
} |
| 477 |
|
| 478 |
macro_rules! define_delimiters { |
| 479 |
($($token:tt pub struct $name:ident #[$doc:meta])*) => { |
| 480 |
$( |
| 481 |
#[$doc] |
| 482 |
pub struct $name { |
| 483 |
pub span: Span, |
| 484 |
} |
| 485 |
|
| 486 |
#[doc(hidden)] |
| 487 |
#[allow(non_snake_case)] |
| 488 |
pub fn $name<S: IntoSpans<[Span; 1]>>(span: S) -> $name { |
| 489 |
$name { |
| 490 |
span: span.into_spans()[0], |
| 491 |
} |
| 492 |
} |
| 493 |
|
| 494 |
impl std::default::Default for $name { |
| 495 |
fn default() -> Self { |
| 496 |
$name { |
| 497 |
span: Span::call_site(), |
| 498 |
} |
| 499 |
} |
| 500 |
} |
| 501 |
|
| 502 |
#[cfg(feature = "clone-impls")] |
| 503 |
#[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))] |
| 504 |
impl Copy for $name {} |
| 505 |
|
| 506 |
#[cfg(feature = "clone-impls")] |
| 507 |
#[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))] |
| 508 |
impl Clone for $name { |
| 509 |
fn clone(&self) -> Self { |
| 510 |
*self |
| 511 |
} |
| 512 |
} |
| 513 |
|
| 514 |
#[cfg(feature = "extra-traits")] |
| 515 |
#[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))] |
| 516 |
impl Debug for $name { |
| 517 |
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
| 518 |
f.write_str(stringify!($name)) |
| 519 |
} |
| 520 |
} |
| 521 |
|
| 522 |
#[cfg(feature = "extra-traits")] |
| 523 |
#[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))] |
| 524 |
impl cmp::Eq for $name {} |
| 525 |
|
| 526 |
#[cfg(feature = "extra-traits")] |
| 527 |
#[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))] |
| 528 |
impl PartialEq for $name { |
| 529 |
fn eq(&self, _other: &$name) -> bool { |
| 530 |
true |
| 531 |
} |
| 532 |
} |
| 533 |
|
| 534 |
#[cfg(feature = "extra-traits")] |
| 535 |
#[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))] |
| 536 |
impl Hash for $name { |
| 537 |
fn hash<H: Hasher>(&self, _state: &mut H) {} |
| 538 |
} |
| 539 |
|
| 540 |
impl $name { |
| 541 |
#[cfg(feature = "printing")] |
| 542 |
pub fn surround<F>(&self, tokens: &mut TokenStream, f: F) |
| 543 |
where |
| 544 |
F: FnOnce(&mut TokenStream), |
| 545 |
{ |
| 546 |
printing::delim($token, self.span, tokens, f); |
| 547 |
} |
| 548 |
} |
| 549 |
|
| 550 |
#[cfg(feature = "parsing")] |
| 551 |
impl private::Sealed for $name {} |
| 552 |
)* |
| 553 |
}; |
| 554 |
} |
| 555 |
|
| 556 |
define_punctuation_structs! { |
| 557 |
"_" pub struct Underscore/1 /// `_` |
| 558 |
} |
| 559 |
|
| 560 |
#[cfg(feature = "printing")] |
| 561 |
#[cfg_attr(doc_cfg, doc(cfg(feature = "printing")))] |
| 562 |
impl ToTokens for Underscore { |
| 563 |
fn to_tokens(&self, tokens: &mut TokenStream) { |
| 564 |
tokens.append(Ident::new("_", self.span)); |
| 565 |
} |
| 566 |
} |
| 567 |
|
| 568 |
#[cfg(feature = "parsing")] |
| 569 |
#[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] |
| 570 |
impl Parse for Underscore { |
| 571 |
fn parse(input: ParseStream) -> Result<Self> { |
| 572 |
input.step(|cursor| { |
| 573 |
if let Some((ident, rest)) = cursor.ident() { |
| 574 |
if ident == "_" { |
| 575 |
return Ok((Underscore(ident.span()), rest)); |
| 576 |
} |
| 577 |
} |
| 578 |
if let Some((punct, rest)) = cursor.punct() { |
| 579 |
if punct.as_char() == '_' { |
| 580 |
return Ok((Underscore(punct.span()), rest)); |
| 581 |
} |
| 582 |
} |
| 583 |
Err(cursor.error("expected `_`")) |
| 584 |
}) |
| 585 |
} |
| 586 |
} |
| 587 |
|
| 588 |
#[cfg(feature = "parsing")] |
| 589 |
impl Token for Underscore { |
| 590 |
fn peek(cursor: Cursor) -> bool { |
| 591 |
if let Some((ident, _rest)) = cursor.ident() { |
| 592 |
return ident == "_"; |
| 593 |
} |
| 594 |
if let Some((punct, _rest)) = cursor.punct() { |
| 595 |
return punct.as_char() == '_'; |
| 596 |
} |
| 597 |
false |
| 598 |
} |
| 599 |
|
| 600 |
fn display() -> &'static str { |
| 601 |
"`_`" |
| 602 |
} |
| 603 |
} |
| 604 |
|
| 605 |
#[cfg(feature = "parsing")] |
| 606 |
impl private::Sealed for Underscore {} |
| 607 |
|
| 608 |
#[cfg(feature = "parsing")] |
| 609 |
impl Token for Paren { |
| 610 |
fn peek(cursor: Cursor) -> bool { |
| 611 |
lookahead::is_delimiter(cursor, Delimiter::Parenthesis) |
| 612 |
} |
| 613 |
|
| 614 |
fn display() -> &'static str { |
| 615 |
"parentheses" |
| 616 |
} |
| 617 |
} |
| 618 |
|
| 619 |
#[cfg(feature = "parsing")] |
| 620 |
impl Token for Brace { |
| 621 |
fn peek(cursor: Cursor) -> bool { |
| 622 |
lookahead::is_delimiter(cursor, Delimiter::Brace) |
| 623 |
} |
| 624 |
|
| 625 |
fn display() -> &'static str { |
| 626 |
"curly braces" |
| 627 |
} |
| 628 |
} |
| 629 |
|
| 630 |
#[cfg(feature = "parsing")] |
| 631 |
impl Token for Bracket { |
| 632 |
fn peek(cursor: Cursor) -> bool { |
| 633 |
lookahead::is_delimiter(cursor, Delimiter::Bracket) |
| 634 |
} |
| 635 |
|
| 636 |
fn display() -> &'static str { |
| 637 |
"square brackets" |
| 638 |
} |
| 639 |
} |
| 640 |
|
| 641 |
#[cfg(feature = "parsing")] |
| 642 |
impl Token for Group { |
| 643 |
fn peek(cursor: Cursor) -> bool { |
| 644 |
lookahead::is_delimiter(cursor, Delimiter::None) |
| 645 |
} |
| 646 |
|
| 647 |
fn display() -> &'static str { |
| 648 |
"invisible group" |
| 649 |
} |
| 650 |
} |
| 651 |
|
| 652 |
define_keywords! { |
| 653 |
"abstract" pub struct Abstract /// `abstract` |
| 654 |
"as" pub struct As /// `as` |
| 655 |
"async" pub struct Async /// `async` |
| 656 |
"auto" pub struct Auto /// `auto` |
| 657 |
"await" pub struct Await /// `await` |
| 658 |
"become" pub struct Become /// `become` |
| 659 |
"box" pub struct Box /// `box` |
| 660 |
"break" pub struct Break /// `break` |
| 661 |
"const" pub struct Const /// `const` |
| 662 |
"continue" pub struct Continue /// `continue` |
| 663 |
"crate" pub struct Crate /// `crate` |
| 664 |
"default" pub struct Default /// `default` |
| 665 |
"do" pub struct Do /// `do` |
| 666 |
"dyn" pub struct Dyn /// `dyn` |
| 667 |
"else" pub struct Else /// `else` |
| 668 |
"enum" pub struct Enum /// `enum` |
| 669 |
"extern" pub struct Extern /// `extern` |
| 670 |
"final" pub struct Final /// `final` |
| 671 |
"fn" pub struct Fn /// `fn` |
| 672 |
"for" pub struct For /// `for` |
| 673 |
"if" pub struct If /// `if` |
| 674 |
"impl" pub struct Impl /// `impl` |
| 675 |
"in" pub struct In /// `in` |
| 676 |
"let" pub struct Let /// `let` |
| 677 |
"loop" pub struct Loop /// `loop` |
| 678 |
"macro" pub struct Macro /// `macro` |
| 679 |
"match" pub struct Match /// `match` |
| 680 |
"mod" pub struct Mod /// `mod` |
| 681 |
"move" pub struct Move /// `move` |
| 682 |
"mut" pub struct Mut /// `mut` |
| 683 |
"override" pub struct Override /// `override` |
| 684 |
"priv" pub struct Priv /// `priv` |
| 685 |
"pub" pub struct Pub /// `pub` |
| 686 |
"ref" pub struct Ref /// `ref` |
| 687 |
"return" pub struct Return /// `return` |
| 688 |
"Self" pub struct SelfType /// `Self` |
| 689 |
"self" pub struct SelfValue /// `self` |
| 690 |
"static" pub struct Static /// `static` |
| 691 |
"struct" pub struct Struct /// `struct` |
| 692 |
"super" pub struct Super /// `super` |
| 693 |
"trait" pub struct Trait /// `trait` |
| 694 |
"try" pub struct Try /// `try` |
| 695 |
"type" pub struct Type /// `type` |
| 696 |
"typeof" pub struct Typeof /// `typeof` |
| 697 |
"union" pub struct Union /// `union` |
| 698 |
"unsafe" pub struct Unsafe /// `unsafe` |
| 699 |
"unsized" pub struct Unsized /// `unsized` |
| 700 |
"use" pub struct Use /// `use` |
| 701 |
"virtual" pub struct Virtual /// `virtual` |
| 702 |
"where" pub struct Where /// `where` |
| 703 |
"while" pub struct While /// `while` |
| 704 |
"yield" pub struct Yield /// `yield` |
| 705 |
} |
| 706 |
|
| 707 |
define_punctuation! { |
| 708 |
"+" pub struct Add/1 /// `+` |
| 709 |
"+=" pub struct AddEq/2 /// `+=` |
| 710 |
"&" pub struct And/1 /// `&` |
| 711 |
"&&" pub struct AndAnd/2 /// `&&` |
| 712 |
"&=" pub struct AndEq/2 /// `&=` |
| 713 |
"@" pub struct At/1 /// `@` |
| 714 |
"!" pub struct Bang/1 /// `!` |
| 715 |
"^" pub struct Caret/1 /// `^` |
| 716 |
"^=" pub struct CaretEq/2 /// `^=` |
| 717 |
":" pub struct Colon/1 /// `:` |
| 718 |
"::" pub struct Colon2/2 /// `::` |
| 719 |
"," pub struct Comma/1 /// `,` |
| 720 |
"/" pub struct Div/1 /// `/` |
| 721 |
"/=" pub struct DivEq/2 /// `/=` |
| 722 |
"$" pub struct Dollar/1 /// `$` |
| 723 |
"." pub struct Dot/1 /// `.` |
| 724 |
".." pub struct Dot2/2 /// `..` |
| 725 |
"..." pub struct Dot3/3 /// `...` |
| 726 |
"..=" pub struct DotDotEq/3 /// `..=` |
| 727 |
"=" pub struct Eq/1 /// `=` |
| 728 |
"==" pub struct EqEq/2 /// `==` |
| 729 |
">=" pub struct Ge/2 /// `>=` |
| 730 |
">" pub struct Gt/1 /// `>` |
| 731 |
"<=" pub struct Le/2 /// `<=` |
| 732 |
"<" pub struct Lt/1 /// `<` |
| 733 |
"*=" pub struct MulEq/2 /// `*=` |
| 734 |
"!=" pub struct Ne/2 /// `!=` |
| 735 |
"|" pub struct Or/1 /// `|` |
| 736 |
"|=" pub struct OrEq/2 /// `|=` |
| 737 |
"||" pub struct OrOr/2 /// `||` |
| 738 |
"#" pub struct Pound/1 /// `#` |
| 739 |
"?" pub struct Question/1 /// `?` |
| 740 |
"->" pub struct RArrow/2 /// `->` |
| 741 |
"<-" pub struct LArrow/2 /// `<-` |
| 742 |
"%" pub struct Rem/1 /// `%` |
| 743 |
"%=" pub struct RemEq/2 /// `%=` |
| 744 |
"=>" pub struct FatArrow/2 /// `=>` |
| 745 |
";" pub struct Semi/1 /// `;` |
| 746 |
"<<" pub struct Shl/2 /// `<<` |
| 747 |
"<<=" pub struct ShlEq/3 /// `<<=` |
| 748 |
">>" pub struct Shr/2 /// `>>` |
| 749 |
">>=" pub struct ShrEq/3 /// `>>=` |
| 750 |
"*" pub struct Star/1 /// `*` |
| 751 |
"-" pub struct Sub/1 /// `-` |
| 752 |
"-=" pub struct SubEq/2 /// `-=` |
| 753 |
"~" pub struct Tilde/1 /// `~` |
| 754 |
} |
| 755 |
|
| 756 |
define_delimiters! { |
| 757 |
"{" pub struct Brace /// `{...}` |
| 758 |
"[" pub struct Bracket /// `[...]` |
| 759 |
"(" pub struct Paren /// `(...)` |
| 760 |
" " pub struct Group /// None-delimited group |
| 761 |
} |
| 762 |
|
| 763 |
macro_rules! export_token_macro { |
| 764 |
($($await_rule:tt)*) => { |
| 765 |
/// A type-macro that expands to the name of the Rust type representation of a |
| 766 |
/// given token. |
| 767 |
/// |
| 768 |
/// See the [token module] documentation for details and examples. |
| 769 |
/// |
| 770 |
/// [token module]: crate::token |
| 771 |
// Unfortunate duplication due to a rustdoc bug. |
| 772 |
// https://github.com/rust-lang/rust/issues/45939 |
| 773 |
#[macro_export] |
| 774 |
macro_rules! Token { |
| 775 |
[abstract] => { $crate::token::Abstract }; |
| 776 |
[as] => { $crate::token::As }; |
| 777 |
[async] => { $crate::token::Async }; |
| 778 |
[auto] => { $crate::token::Auto }; |
| 779 |
$($await_rule => { $crate::token::Await };)* |
| 780 |
[become] => { $crate::token::Become }; |
| 781 |
[box] => { $crate::token::Box }; |
| 782 |
[break] => { $crate::token::Break }; |
| 783 |
[const] => { $crate::token::Const }; |
| 784 |
[continue] => { $crate::token::Continue }; |
| 785 |
[crate] => { $crate::token::Crate }; |
| 786 |
[default] => { $crate::token::Default }; |
| 787 |
[do] => { $crate::token::Do }; |
| 788 |
[dyn] => { $crate::token::Dyn }; |
| 789 |
[else] => { $crate::token::Else }; |
| 790 |
[enum] => { $crate::token::Enum }; |
| 791 |
[extern] => { $crate::token::Extern }; |
| 792 |
[final] => { $crate::token::Final }; |
| 793 |
[fn] => { $crate::token::Fn }; |
| 794 |
[for] => { $crate::token::For }; |
| 795 |
[if] => { $crate::token::If }; |
| 796 |
[impl] => { $crate::token::Impl }; |
| 797 |
[in] => { $crate::token::In }; |
| 798 |
[let] => { $crate::token::Let }; |
| 799 |
[loop] => { $crate::token::Loop }; |
| 800 |
[macro] => { $crate::token::Macro }; |
| 801 |
[match] => { $crate::token::Match }; |
| 802 |
[mod] => { $crate::token::Mod }; |
| 803 |
[move] => { $crate::token::Move }; |
| 804 |
[mut] => { $crate::token::Mut }; |
| 805 |
[override] => { $crate::token::Override }; |
| 806 |
[priv] => { $crate::token::Priv }; |
| 807 |
[pub] => { $crate::token::Pub }; |
| 808 |
[ref] => { $crate::token::Ref }; |
| 809 |
[return] => { $crate::token::Return }; |
| 810 |
[Self] => { $crate::token::SelfType }; |
| 811 |
[self] => { $crate::token::SelfValue }; |
| 812 |
[static] => { $crate::token::Static }; |
| 813 |
[struct] => { $crate::token::Struct }; |
| 814 |
[super] => { $crate::token::Super }; |
| 815 |
[trait] => { $crate::token::Trait }; |
| 816 |
[try] => { $crate::token::Try }; |
| 817 |
[type] => { $crate::token::Type }; |
| 818 |
[typeof] => { $crate::token::Typeof }; |
| 819 |
[union] => { $crate::token::Union }; |
| 820 |
[unsafe] => { $crate::token::Unsafe }; |
| 821 |
[unsized] => { $crate::token::Unsized }; |
| 822 |
[use] => { $crate::token::Use }; |
| 823 |
[virtual] => { $crate::token::Virtual }; |
| 824 |
[where] => { $crate::token::Where }; |
| 825 |
[while] => { $crate::token::While }; |
| 826 |
[yield] => { $crate::token::Yield }; |
| 827 |
[+] => { $crate::token::Add }; |
| 828 |
[+=] => { $crate::token::AddEq }; |
| 829 |
[&] => { $crate::token::And }; |
| 830 |
[&&] => { $crate::token::AndAnd }; |
| 831 |
[&=] => { $crate::token::AndEq }; |
| 832 |
[@] => { $crate::token::At }; |
| 833 |
[!] => { $crate::token::Bang }; |
| 834 |
[^] => { $crate::token::Caret }; |
| 835 |
[^=] => { $crate::token::CaretEq }; |
| 836 |
[:] => { $crate::token::Colon }; |
| 837 |
[::] => { $crate::token::Colon2 }; |
| 838 |
[,] => { $crate::token::Comma }; |
| 839 |
[/] => { $crate::token::Div }; |
| 840 |
[/=] => { $crate::token::DivEq }; |
| 841 |
[$] => { $crate::token::Dollar }; |
| 842 |
[.] => { $crate::token::Dot }; |
| 843 |
[..] => { $crate::token::Dot2 }; |
| 844 |
[...] => { $crate::token::Dot3 }; |
| 845 |
[..=] => { $crate::token::DotDotEq }; |
| 846 |
[=] => { $crate::token::Eq }; |
| 847 |
[==] => { $crate::token::EqEq }; |
| 848 |
[>=] => { $crate::token::Ge }; |
| 849 |
[>] => { $crate::token::Gt }; |
| 850 |
[<=] => { $crate::token::Le }; |
| 851 |
[<] => { $crate::token::Lt }; |
| 852 |
[*=] => { $crate::token::MulEq }; |
| 853 |
[!=] => { $crate::token::Ne }; |
| 854 |
[|] => { $crate::token::Or }; |
| 855 |
[|=] => { $crate::token::OrEq }; |
| 856 |
[||] => { $crate::token::OrOr }; |
| 857 |
[#] => { $crate::token::Pound }; |
| 858 |
[?] => { $crate::token::Question }; |
| 859 |
[->] => { $crate::token::RArrow }; |
| 860 |
[<-] => { $crate::token::LArrow }; |
| 861 |
[%] => { $crate::token::Rem }; |
| 862 |
[%=] => { $crate::token::RemEq }; |
| 863 |
[=>] => { $crate::token::FatArrow }; |
| 864 |
[;] => { $crate::token::Semi }; |
| 865 |
[<<] => { $crate::token::Shl }; |
| 866 |
[<<=] => { $crate::token::ShlEq }; |
| 867 |
[>>] => { $crate::token::Shr }; |
| 868 |
[>>=] => { $crate::token::ShrEq }; |
| 869 |
[*] => { $crate::token::Star }; |
| 870 |
[-] => { $crate::token::Sub }; |
| 871 |
[-=] => { $crate::token::SubEq }; |
| 872 |
[~] => { $crate::token::Tilde }; |
| 873 |
[_] => { $crate::token::Underscore }; |
| 874 |
} |
| 875 |
}; |
| 876 |
} |
| 877 |
|
| 878 |
// Old rustc does not permit `await` appearing anywhere in the source file. |
| 879 |
// https://github.com/rust-lang/rust/issues/57919 |
| 880 |
// We put the Token![await] rule in a place that is not lexed by old rustc. |
| 881 |
#[cfg(not(syn_omit_await_from_token_macro))] |
| 882 |
include!("await.rs"); // export_token_macro! {[await]} |
| 883 |
#[cfg(syn_omit_await_from_token_macro)] |
| 884 |
export_token_macro! {} |
| 885 |
|
| 886 |
// Not public API. |
| 887 |
#[doc(hidden)] |
| 888 |
#[cfg(feature = "parsing")] |
| 889 |
pub mod parsing { |
| 890 |
use crate::buffer::Cursor; |
| 891 |
use crate::error::{Error, Result}; |
| 892 |
use crate::parse::ParseStream; |
| 893 |
use crate::span::FromSpans; |
| 894 |
use proc_macro2::{Spacing, Span}; |
| 895 |
|
| 896 |
pub fn keyword(input: ParseStream, token: &str) -> Result<Span> { |
| 897 |
input.step(|cursor| { |
| 898 |
if let Some((ident, rest)) = cursor.ident() { |
| 899 |
if ident == token { |
| 900 |
return Ok((ident.span(), rest)); |
| 901 |
} |
| 902 |
} |
| 903 |
Err(cursor.error(format!("expected `{}`", token))) |
| 904 |
}) |
| 905 |
} |
| 906 |
|
| 907 |
pub fn peek_keyword(cursor: Cursor, token: &str) -> bool { |
| 908 |
if let Some((ident, _rest)) = cursor.ident() { |
| 909 |
ident == token |
| 910 |
} else { |
| 911 |
false |
| 912 |
} |
| 913 |
} |
| 914 |
|
| 915 |
pub fn punct<S: FromSpans>(input: ParseStream, token: &str) -> Result<S> { |
| 916 |
let mut spans = [input.span(); 3]; |
| 917 |
punct_helper(input, token, &mut spans)?; |
| 918 |
Ok(S::from_spans(&spans)) |
| 919 |
} |
| 920 |
|
| 921 |
fn punct_helper(input: ParseStream, token: &str, spans: &mut [Span; 3]) -> Result<()> { |
| 922 |
input.step(|cursor| { |
| 923 |
let mut cursor = *cursor; |
| 924 |
assert!(token.len() <= spans.len()); |
| 925 |
|
| 926 |
for (i, ch) in token.chars().enumerate() { |
| 927 |
match cursor.punct() { |
| 928 |
Some((punct, rest)) => { |
| 929 |
spans[i] = punct.span(); |
| 930 |
if punct.as_char() != ch { |
| 931 |
break; |
| 932 |
} else if i == token.len() - 1 { |
| 933 |
return Ok(((), rest)); |
| 934 |
} else if punct.spacing() != Spacing::Joint { |
| 935 |
break; |
| 936 |
} |
| 937 |
cursor = rest; |
| 938 |
} |
| 939 |
None => break, |
| 940 |
} |
| 941 |
} |
| 942 |
|
| 943 |
Err(Error::new(spans[0], format!("expected `{}`", token))) |
| 944 |
}) |
| 945 |
} |
| 946 |
|
| 947 |
pub fn peek_punct(mut cursor: Cursor, token: &str) -> bool { |
| 948 |
for (i, ch) in token.chars().enumerate() { |
| 949 |
match cursor.punct() { |
| 950 |
Some((punct, rest)) => { |
| 951 |
if punct.as_char() != ch { |
| 952 |
break; |
| 953 |
} else if i == token.len() - 1 { |
| 954 |
return true; |
| 955 |
} else if punct.spacing() != Spacing::Joint { |
| 956 |
break; |
| 957 |
} |
| 958 |
cursor = rest; |
| 959 |
} |
| 960 |
None => break, |
| 961 |
} |
| 962 |
} |
| 963 |
false |
| 964 |
} |
| 965 |
} |
| 966 |
|
| 967 |
// Not public API. |
| 968 |
#[doc(hidden)] |
| 969 |
#[cfg(feature = "printing")] |
| 970 |
pub mod printing { |
| 971 |
use proc_macro2::{Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream}; |
| 972 |
use quote::TokenStreamExt; |
| 973 |
|
| 974 |
pub fn punct(s: &str, spans: &[Span], tokens: &mut TokenStream) { |
| 975 |
assert_eq!(s.len(), spans.len()); |
| 976 |
|
| 977 |
let mut chars = s.chars(); |
| 978 |
let mut spans = spans.iter(); |
| 979 |
let ch = chars.next_back().unwrap(); |
| 980 |
let span = spans.next_back().unwrap(); |
| 981 |
for (ch, span) in chars.zip(spans) { |
| 982 |
let mut op = Punct::new(ch, Spacing::Joint); |
| 983 |
op.set_span(*span); |
| 984 |
tokens.append(op); |
| 985 |
} |
| 986 |
|
| 987 |
let mut op = Punct::new(ch, Spacing::Alone); |
| 988 |
op.set_span(*span); |
| 989 |
tokens.append(op); |
| 990 |
} |
| 991 |
|
| 992 |
pub fn keyword(s: &str, span: Span, tokens: &mut TokenStream) { |
| 993 |
tokens.append(Ident::new(s, span)); |
| 994 |
} |
| 995 |
|
| 996 |
pub fn delim<F>(s: &str, span: Span, tokens: &mut TokenStream, f: F) |
| 997 |
where |
| 998 |
F: FnOnce(&mut TokenStream), |
| 999 |
{ |
| 1000 |
let delim = match s { |
| 1001 |
"(" => Delimiter::Parenthesis, |
| 1002 |
"[" => Delimiter::Bracket, |
| 1003 |
"{" => Delimiter::Brace, |
| 1004 |
" " => Delimiter::None, |
| 1005 |
_ => panic!("unknown delimiter: {}", s), |
| 1006 |
}; |
| 1007 |
let mut inner = TokenStream::new(); |
| 1008 |
f(&mut inner); |
| 1009 |
let mut g = Group::new(delim, inner); |
| 1010 |
g.set_span(span); |
| 1011 |
tokens.append(g); |
| 1012 |
} |
| 1013 |
} |