Debian Codegraph @

llvm-project/clang-tools-extra/pseudo/include/clang-pseudo/Token.h

raw
1 //===--- Token.h - Tokens and token streams in the pseudoparser --*- C++-*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Tokens are the first level of abstraction above bytes used in pseudoparsing.
10 // We use clang's lexer to scan the bytes (in raw mode, with no preprocessor).
11 // The tokens is wrapped into pseudo::Token, along with line/indent info.
12 //
13 // Unlike clang, we make multiple passes over the whole file, out-of-order.
14 // Therefore we retain the whole token sequence in memory. (This is feasible as
15 // we process one file at a time). pseudo::TokenStream holds such a stream.
16 // The initial stream holds the raw tokens read from the file, later passes
17 // operate on derived TokenStreams (e.g. with directives stripped).
18 //
19 // Similar facilities from clang that are *not* used:
20 // - SourceManager: designed around multiple files and precise macro expansion.
21 // - clang::Token: coupled to SourceManager, doesn't retain layout info.
22 // (pseudo::Token is similar, but without SourceLocations).
23 // - syntax::TokenBuffer: coupled to SourceManager, has #includes and macros.
24 // (pseudo::TokenStream is similar, but a flat token list).
25 //
26 //===----------------------------------------------------------------------===//
27
28 #ifndef CLANG_PSEUDO_TOKEN_H
29 #define CLANG_PSEUDO_TOKEN_H
30
31 #include "clang/Basic/LLVM.h"
32 #include "clang/Basic/LangStandard.h"
33 #include "clang/Basic/TokenKinds.h"
34 #include "llvm/ADT/ArrayRef.h"
35 #include "llvm/Support/raw_ostream.h"
36 #include <cstdint>
37 #include <limits>
38 #include <memory>
39 #include <vector>
40
41 namespace clang {
42 class LangOptions;
43 namespace pseudo {
44
45 /// A single C++ or preprocessor token.
46 ///
47 /// Unlike clang::Token and syntax::Token, these tokens are not connected to a
48 /// SourceManager - we are not dealing with multiple files.
49 struct Token {
50 /// An Index identifies a token within a stream.
51 using Index = uint32_t;
52 /// A sentinel Index indicating no token.
53 constexpr static Index Invalid = std::numeric_limits<Index>::max();
54 struct Range;
55
56 /// The token text.
57 ///
58 /// Typically from the original source file, but may have been synthesized.
59 StringRef text() const { return StringRef(Data, Length); }
60 const char *Data = nullptr;
61 uint32_t Length = 0;
62
63 /// Zero-based line number for the start of the token.
64 /// This refers to the original source file as written.
65 uint32_t Line = 0;
66 /// Width of whitespace before the first token on this line.
67 uint8_t Indent = 0;
68 /// Flags have some meaning defined by the function that produced this stream.
69 uint8_t Flags = 0;
70 /// Index into the original token stream (as raw-lexed from the source code).
71 Index OriginalIndex = Invalid;
72 // Helpers to get/set Flags based on `enum class`.
73 template <class T> bool flag(T Mask) const {
74 return Flags & uint8_t{static_cast<std::underlying_type_t<T>>(Mask)};
75 }
76 template <class T> void setFlag(T Mask) {
77 Flags |= uint8_t{static_cast<std::underlying_type_t<T>>(Mask)};
78 }
79
80 /// Returns the next token in the stream. this may not be a sentinel.
81 const Token &next() const {
82 assert(Kind != tok::eof);
83 return *(this + 1);
84 }
85 /// Returns the next token in the stream, skipping over comments.
86 const Token &nextNC() const {
87 const Token *T = this;
88 do
89 T = &T->next();
90 while (T->Kind == tok::comment);
91 return *T;
92 }
93 /// Returns the previous token in the stream. this may not be a sentinel.
94 const Token &prev() const {
95 assert(Kind != tok::eof);
96 return *(this - 1);
97 }
98 /// Returns the bracket paired with this one, if any.
99 const Token *pair() const { return Pair == 0 ? nullptr : this + Pair; }
100
101 /// The type of token as determined by clang's lexer.
102 clang::tok::TokenKind Kind = clang::tok::unknown;
103 /// If this token is a paired bracket, the offset of the pair in the stream.
104 int32_t Pair = 0;
105 };
106 static_assert(sizeof(Token) <= sizeof(char *) + 24, "Careful with layout!");
107 llvm::raw_ostream &operator<<(llvm::raw_ostream &, const Token &);
108
109 /// A half-open range of tokens within a stream.
110 struct Token::Range {
111 Index Begin = 0;
112 Index End = 0;
113
114 uint32_t size() const { return End - Begin; }
115 static Range emptyAt(Index Index) { return Range{Index, Index}; }
116 };
117 llvm::raw_ostream &operator<<(llvm::raw_ostream &, const Token::Range &);
118
119 /// A complete sequence of Tokens representing a source file.
120 ///
121 /// This may match a raw file from disk, or be derived from a previous stream.
122 /// For example, stripping comments from a TokenStream results in a new stream.
123 ///
124 /// A stream has sentinel 'eof' tokens at each end, e.g `int main();` becomes:
125 /// int main ( ) ;
126 /// eof kw_int ident l_paren r_paren semi eof
127 /// front() back()
128 /// 0 1 2 3 4 5
129 class TokenStream {
130 public:
131 /// Create an empty stream.
132 ///
133 /// Initially, the stream is appendable and not finalized.
134 /// The token sequence may only be accessed after finalize() is called.
135 ///
136 /// Payload is an opaque object which will be owned by the stream.
137 /// e.g. an allocator to hold backing storage for synthesized token text.
138 explicit TokenStream(std::shared_ptr<void> Payload = nullptr);
139
140 /// Append a token to the stream, which must not be finalized.
141 void push(Token T) {
142 assert(!isFinalized());
143 Storage.push_back(std::move(T));
144 }
145
146 /// Finalize the token stream, allowing tokens to be accessed.
147 /// Tokens may no longer be appended.
148 void finalize();
149 bool isFinalized() const;
150
151 /// Returns the index of T within the stream.
152 ///
153 /// T must be within the stream or the end sentinel (not the start sentinel).
154 Token::Index index(const Token &T) const {
155 assert(isFinalized());
156 assert(&T >= Storage.data() && &T < Storage.data() + Storage.size());
157 assert(&T != Storage.data() && "start sentinel");
158 return &T - Tokens.data();
159 }
160
161 ArrayRef<Token> tokens() const {
162 assert(isFinalized());
163 return Tokens;
164 }
165 ArrayRef<Token> tokens(Token::Range R) const {
166 return tokens().slice(R.Begin, R.End - R.Begin);
167 }
168
169 MutableArrayRef<Token> tokens() {
170 assert(isFinalized());
171 return Tokens;
172 }
173
174 /// May return the end sentinel if the stream is empty.
175 const Token &front() const {
176 assert(isFinalized());
177 return Storage[1];
178 }
179
180 /// Returns the shared payload.
181 std::shared_ptr<void> getPayload() const { return Payload; }
182 /// Adds the given payload to the stream.
183 void addPayload(std::shared_ptr<void> P) {
184 if (!Payload)
185 Payload = std::move(P);
186 else
187 Payload = std::make_shared<
188 std::pair<std::shared_ptr<void>, std::shared_ptr<void>>>(
189 std::move(P), std::move(Payload));
190 }
191
192 /// Print the tokens in this stream to the output stream.
193 ///
194 /// The presence of newlines/spaces is preserved, but not the quantity.
195 void print(llvm::raw_ostream &) const;
196
197 private:
198 std::shared_ptr<void> Payload;
199
200 MutableArrayRef<Token> Tokens;
201 std::vector<Token> Storage; // eof + Tokens + eof
202 };
203 llvm::raw_ostream &operator<<(llvm::raw_ostream &, const TokenStream &);
204
205 /// Extracts a raw token stream from the source code.
206 ///
207 /// All tokens will reference the data of the provided string.
208 /// "word-like" tokens such as identifiers and keywords will be raw_identifier.
209 TokenStream lex(const std::string &, const clang::LangOptions &);
210 enum class LexFlags : uint8_t {
211 /// Marks the token at the start of a logical preprocessor line.
212 /// This is a position where a directive might start.
213 ///
214 /// Here, the first # is StartsPPLine, but second is not (same logical line).
215 /// #define X(error) \
216 /// #error // not a directive!
217 ///
218 /// Careful, the directive may not start exactly on the StartsPPLine token:
219 /// /*comment*/ #include <foo.h>
220 StartsPPLine = 1 << 0,
221 /// Marks tokens containing trigraphs, escaped newlines, UCNs etc.
222 /// The text() of such tokens will contain the raw trigrah.
223 NeedsCleaning = 1 << 1,
224 };
225 /// A generic lang options suitable for lexing/parsing a langage.
226 clang::LangOptions genericLangOpts(
227 clang::Language = clang::Language::CXX,
228 clang::LangStandard::Kind = clang::LangStandard::lang_unspecified);
229
230 /// Decoding raw tokens written in the source code, returning a derived stream.
231 ///
232 /// - escaped newlines within tokens are removed
233 /// - trigraphs are replaced with the characters they encode
234 /// - UCNs within raw_identifiers are replaced by the characters they encode
235 /// (UCNs within strings, comments etc are not translated)
236 /// - raw_identifier tokens are assigned their correct keyword type
237 /// - the >> token is split into separate > > tokens
238 /// (we use a modified grammar where >> is a nonterminal, not a token)
239 ///
240 /// The StartsPPLine flag is preserved.
241 ///
242 /// Formally the identifier correctly happens before preprocessing, while we
243 /// should only cook raw_identifiers that survive preprocessing.
244 /// However, ignoring the Token::Kind of tokens in directives achieves the same.
245 /// (And having cooked token kinds in PP-disabled sections is useful for us).
246 TokenStream cook(const TokenStream &, const clang::LangOptions &);
247
248 /// Drops comment tokens.
249 TokenStream stripComments(const TokenStream &);
250
251 } // namespace pseudo
252 } // namespace clang
253
254 #endif // CLANG_PSEUDO_TOKEN_H