| 1 |
// (C) Copyright 2016 Jethro G. Beekman |
| 2 |
// |
| 3 |
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or |
| 4 |
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license |
| 5 |
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your |
| 6 |
// option. This file may not be copied, modified, or distributed |
| 7 |
// except according to those terms. |
| 8 |
//! Representation of a C token |
| 9 |
//! |
| 10 |
//! This is designed to map onto a libclang CXToken. |
| 11 |
|
| 12 |
#[derive(Debug, Copy, Clone, PartialEq, Eq)] |
| 13 |
#[allow(missing_docs)] |
| 14 |
pub enum Kind { |
| 15 |
Punctuation, |
| 16 |
Keyword, |
| 17 |
Identifier, |
| 18 |
Literal, |
| 19 |
Comment, |
| 20 |
} |
| 21 |
|
| 22 |
/// A single token in a C expression. |
| 23 |
#[derive(Debug, Clone, PartialEq, Eq)] |
| 24 |
pub struct Token { |
| 25 |
/// The type of this token. |
| 26 |
pub kind: Kind, |
| 27 |
/// The bytes that make up the token. |
| 28 |
pub raw: Box<[u8]>, |
| 29 |
} |
| 30 |
|
| 31 |
impl<'a> From<(Kind, &'a [u8])> for Token { |
| 32 |
fn from((kind, value): (Kind, &'a [u8])) -> Token { |
| 33 |
Token { |
| 34 |
kind, |
| 35 |
raw: value.to_owned().into_boxed_slice(), |
| 36 |
} |
| 37 |
} |
| 38 |
} |
| 39 |
|
| 40 |
/// Remove all comment tokens from a vector of tokens |
| 41 |
pub fn remove_comments(v: &mut Vec<Token>) -> &mut Vec<Token> { |
| 42 |
v.retain(|t| t.kind != Kind::Comment); |
| 43 |
v |
| 44 |
} |