Debian Codegraph @

llvm-project/clang-tools-extra/clangd/SemanticHighlighting.cpp

raw
1 //===--- SemanticHighlighting.cpp - ------------------------- ---*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "SemanticHighlighting.h"
10 #include "FindTarget.h"
11 #include "HeuristicResolver.h"
12 #include "ParsedAST.h"
13 #include "Protocol.h"
14 #include "SourceCode.h"
15 #include "support/Logger.h"
16 #include "clang/AST/ASTContext.h"
17 #include "clang/AST/Decl.h"
18 #include "clang/AST/DeclCXX.h"
19 #include "clang/AST/DeclObjC.h"
20 #include "clang/AST/DeclTemplate.h"
21 #include "clang/AST/DeclarationName.h"
22 #include "clang/AST/ExprCXX.h"
23 #include "clang/AST/RecursiveASTVisitor.h"
24 #include "clang/AST/Type.h"
25 #include "clang/AST/TypeLoc.h"
26 #include "clang/Basic/LangOptions.h"
27 #include "clang/Basic/SourceLocation.h"
28 #include "clang/Basic/SourceManager.h"
29 #include "clang/Tooling/Syntax/Tokens.h"
30 #include "llvm/ADT/STLExtras.h"
31 #include "llvm/ADT/StringRef.h"
32 #include "llvm/Support/Casting.h"
33 #include "llvm/Support/Error.h"
34 #include <algorithm>
35 #include <optional>
36
37 namespace clang {
38 namespace clangd {
39 namespace {
40
41 /// Some names are not written in the source code and cannot be highlighted,
42 /// e.g. anonymous classes. This function detects those cases.
43 bool canHighlightName(DeclarationName Name) {
44 switch (Name.getNameKind()) {
45 case DeclarationName::Identifier: {
46 auto *II = Name.getAsIdentifierInfo();
47 return II && !II->getName().empty();
48 }
49 case DeclarationName::CXXConstructorName:
50 case DeclarationName::CXXDestructorName:
51 return true;
52 case DeclarationName::ObjCZeroArgSelector:
53 case DeclarationName::ObjCOneArgSelector:
54 case DeclarationName::ObjCMultiArgSelector:
55 // Multi-arg selectors need special handling, and we handle 0/1 arg
56 // selectors there too.
57 return false;
58 case DeclarationName::CXXConversionFunctionName:
59 case DeclarationName::CXXOperatorName:
60 case DeclarationName::CXXDeductionGuideName:
61 case DeclarationName::CXXLiteralOperatorName:
62 case DeclarationName::CXXUsingDirective:
63 return false;
64 }
65 llvm_unreachable("invalid name kind");
66 }
67
68 bool isUniqueDefinition(const NamedDecl *Decl) {
69 if (auto *Func = dyn_cast<FunctionDecl>(Decl))
70 return Func->isThisDeclarationADefinition();
71 if (auto *Klass = dyn_cast<CXXRecordDecl>(Decl))
72 return Klass->isThisDeclarationADefinition();
73 if (auto *Iface = dyn_cast<ObjCInterfaceDecl>(Decl))
74 return Iface->isThisDeclarationADefinition();
75 if (auto *Proto = dyn_cast<ObjCProtocolDecl>(Decl))
76 return Proto->isThisDeclarationADefinition();
77 if (auto *Var = dyn_cast<VarDecl>(Decl))
78 return Var->isThisDeclarationADefinition();
79 return isa<TemplateTypeParmDecl>(Decl) ||
80 isa<NonTypeTemplateParmDecl>(Decl) ||
81 isa<TemplateTemplateParmDecl>(Decl) || isa<ObjCCategoryDecl>(Decl) ||
82 isa<ObjCImplDecl>(Decl);
83 }
84
85 std::optional<HighlightingKind> kindForType(const Type *TP,
86 const HeuristicResolver *Resolver);
87 std::optional<HighlightingKind> kindForDecl(const NamedDecl *D,
88 const HeuristicResolver *Resolver) {
89 if (auto *USD = dyn_cast<UsingShadowDecl>(D)) {
90 if (auto *Target = USD->getTargetDecl())
91 D = Target;
92 }
93 if (auto *TD = dyn_cast<TemplateDecl>(D)) {
94 if (auto *Templated = TD->getTemplatedDecl())
95 D = Templated;
96 }
97 if (auto *TD = dyn_cast<TypedefNameDecl>(D)) {
98 // We try to highlight typedefs as their underlying type.
99 if (auto K =
100 kindForType(TD->getUnderlyingType().getTypePtrOrNull(), Resolver))
101 return K;
102 // And fallback to a generic kind if this fails.
103 return HighlightingKind::Typedef;
104 }
105 // We highlight class decls, constructor decls and destructor decls as
106 // `Class` type. The destructor decls are handled in `VisitTagTypeLoc` (we
107 // will visit a TypeLoc where the underlying Type is a CXXRecordDecl).
108 if (auto *RD = llvm::dyn_cast<RecordDecl>(D)) {
109 // We don't want to highlight lambdas like classes.
110 if (RD->isLambda())
111 return std::nullopt;
112 return HighlightingKind::Class;
113 }
114 if (isa<ClassTemplateDecl, RecordDecl, CXXConstructorDecl, ObjCInterfaceDecl,
115 ObjCImplementationDecl>(D))
116 return HighlightingKind::Class;
117 if (isa<ObjCProtocolDecl>(D))
118 return HighlightingKind::Interface;
119 if (isa<ObjCCategoryDecl>(D))
120 return HighlightingKind::Namespace;
121 if (auto *MD = dyn_cast<CXXMethodDecl>(D))
122 return MD->isStatic() ? HighlightingKind::StaticMethod
123 : HighlightingKind::Method;
124 if (auto *OMD = dyn_cast<ObjCMethodDecl>(D))
125 return OMD->isClassMethod() ? HighlightingKind::StaticMethod
126 : HighlightingKind::Method;
127 if (isa<FieldDecl, ObjCPropertyDecl>(D))
128 return HighlightingKind::Field;
129 if (isa<EnumDecl>(D))
130 return HighlightingKind::Enum;
131 if (isa<EnumConstantDecl>(D))
132 return HighlightingKind::EnumConstant;
133 if (isa<ParmVarDecl>(D))
134 return HighlightingKind::Parameter;
135 if (auto *VD = dyn_cast<VarDecl>(D)) {
136 if (isa<ImplicitParamDecl>(VD)) // e.g. ObjC Self
137 return std::nullopt;
138 return VD->isStaticDataMember()
139 ? HighlightingKind::StaticField
140 : VD->isLocalVarDecl() ? HighlightingKind::LocalVariable
141 : HighlightingKind::Variable;
142 }
143 if (const auto *BD = dyn_cast<BindingDecl>(D))
144 return BD->getDeclContext()->isFunctionOrMethod()
145 ? HighlightingKind::LocalVariable
146 : HighlightingKind::Variable;
147 if (isa<FunctionDecl>(D))
148 return HighlightingKind::Function;
149 if (isa<NamespaceDecl>(D) || isa<NamespaceAliasDecl>(D) ||
150 isa<UsingDirectiveDecl>(D))
151 return HighlightingKind::Namespace;
152 if (isa<TemplateTemplateParmDecl>(D) || isa<TemplateTypeParmDecl>(D) ||
153 isa<NonTypeTemplateParmDecl>(D))
154 return HighlightingKind::TemplateParameter;
155 if (isa<ConceptDecl>(D))
156 return HighlightingKind::Concept;
157 if (const auto *UUVD = dyn_cast<UnresolvedUsingValueDecl>(D)) {
158 auto Targets = Resolver->resolveUsingValueDecl(UUVD);
159 if (!Targets.empty()) {
160 return kindForDecl(Targets[0], Resolver);
161 }
162 return HighlightingKind::Unknown;
163 }
164 return std::nullopt;
165 }
166 std::optional<HighlightingKind> kindForType(const Type *TP,
167 const HeuristicResolver *Resolver) {
168 if (!TP)
169 return std::nullopt;
170 if (TP->isBuiltinType()) // Builtins are special, they do not have decls.
171 return HighlightingKind::Primitive;
172 if (auto *TD = dyn_cast<TemplateTypeParmType>(TP))
173 return kindForDecl(TD->getDecl(), Resolver);
174 if (isa<ObjCObjectPointerType>(TP))
175 return HighlightingKind::Class;
176 if (auto *TD = TP->getAsTagDecl())
177 return kindForDecl(TD, Resolver);
178 return std::nullopt;
179 }
180
181 // Whether T is const in a loose sense - is a variable with this type readonly?
182 bool isConst(QualType T) {
183 if (T.isNull())
184 return false;
185 T = T.getNonReferenceType();
186 if (T.isConstQualified())
187 return true;
188 if (const auto *AT = T->getAsArrayTypeUnsafe())
189 return isConst(AT->getElementType());
190 if (isConst(T->getPointeeType()))
191 return true;
192 return false;
193 }
194
195 // Whether D is const in a loose sense (should it be highlighted as such?)
196 // FIXME: This is separate from whether *a particular usage* can mutate D.
197 // We may want V in V.size() to be readonly even if V is mutable.
198 bool isConst(const Decl *D) {
199 if (llvm::isa<EnumConstantDecl>(D) || llvm::isa<NonTypeTemplateParmDecl>(D))
200 return true;
201 if (llvm::isa<FieldDecl>(D) || llvm::isa<VarDecl>(D) ||
202 llvm::isa<MSPropertyDecl>(D) || llvm::isa<BindingDecl>(D)) {
203 if (isConst(llvm::cast<ValueDecl>(D)->getType()))
204 return true;
205 }
206 if (const auto *OCPD = llvm::dyn_cast<ObjCPropertyDecl>(D)) {
207 if (OCPD->isReadOnly())
208 return true;
209 }
210 if (const auto *MPD = llvm::dyn_cast<MSPropertyDecl>(D)) {
211 if (!MPD->hasSetter())
212 return true;
213 }
214 if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D)) {
215 if (CMD->isConst())
216 return true;
217 }
218 return false;
219 }
220
221 // "Static" means many things in C++, only some get the "static" modifier.
222 //
223 // Meanings that do:
224 // - Members associated with the class rather than the instance.
225 // This is what 'static' most often means across languages.
226 // - static local variables
227 // These are similarly "detached from their context" by the static keyword.
228 // In practice, these are rarely used inside classes, reducing confusion.
229 //
230 // Meanings that don't:
231 // - Namespace-scoped variables, which have static storage class.
232 // This is implicit, so the keyword "static" isn't so strongly associated.
233 // If we want a modifier for these, "global scope" is probably the concept.
234 // - Namespace-scoped variables/functions explicitly marked "static".
235 // There the keyword changes *linkage* , which is a totally different concept.
236 // If we want to model this, "file scope" would be a nice modifier.
237 //
238 // This is confusing, and maybe we should use another name, but because "static"
239 // is a standard LSP modifier, having one with that name has advantages.
240 bool isStatic(const Decl *D) {
241 if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D))
242 return CMD->isStatic();
243 if (const VarDecl *VD = llvm::dyn_cast<VarDecl>(D))
244 return VD->isStaticDataMember() || VD->isStaticLocal();
245 if (const auto *OPD = llvm::dyn_cast<ObjCPropertyDecl>(D))
246 return OPD->isClassProperty();
247 if (const auto *OMD = llvm::dyn_cast<ObjCMethodDecl>(D))
248 return OMD->isClassMethod();
249 return false;
250 }
251
252 bool isAbstract(const Decl *D) {
253 if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D))
254 return CMD->isPure();
255 if (const auto *CRD = llvm::dyn_cast<CXXRecordDecl>(D))
256 return CRD->hasDefinition() && CRD->isAbstract();
257 return false;
258 }
259
260 bool isVirtual(const Decl *D) {
261 if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D))
262 return CMD->isVirtual();
263 return false;
264 }
265
266 bool isDependent(const Decl *D) {
267 if (isa<UnresolvedUsingValueDecl>(D))
268 return true;
269 return false;
270 }
271
272 /// Returns true if `Decl` is considered to be from a default/system library.
273 /// This currently checks the systemness of the file by include type, although
274 /// different heuristics may be used in the future (e.g. sysroot paths).
275 bool isDefaultLibrary(const Decl *D) {
276 SourceLocation Loc = D->getLocation();
277 if (!Loc.isValid())
278 return false;
279 return D->getASTContext().getSourceManager().isInSystemHeader(Loc);
280 }
281
282 bool isDefaultLibrary(const Type *T) {
283 if (!T)
284 return false;
285 const Type *Underlying = T->getPointeeOrArrayElementType();
286 if (Underlying->isBuiltinType())
287 return true;
288 if (auto *TD = dyn_cast<TemplateTypeParmType>(Underlying))
289 return isDefaultLibrary(TD->getDecl());
290 if (auto *TD = Underlying->getAsTagDecl())
291 return isDefaultLibrary(TD);
292 return false;
293 }
294
295 // For a macro usage `DUMP(foo)`, we want:
296 // - DUMP --> "macro"
297 // - foo --> "variable".
298 SourceLocation getHighlightableSpellingToken(SourceLocation L,
299 const SourceManager &SM) {
300 if (L.isFileID())
301 return SM.isWrittenInMainFile(L) ? L : SourceLocation{};
302 // Tokens expanded from the macro body contribute no highlightings.
303 if (!SM.isMacroArgExpansion(L))
304 return {};
305 // Tokens expanded from macro args are potentially highlightable.
306 return getHighlightableSpellingToken(SM.getImmediateSpellingLoc(L), SM);
307 }
308
309 unsigned evaluateHighlightPriority(const HighlightingToken &Tok) {
310 enum HighlightPriority { Dependent = 0, Resolved = 1 };
311 return (Tok.Modifiers & (1 << uint32_t(HighlightingModifier::DependentName)))
312 ? Dependent
313 : Resolved;
314 }
315
316 // Sometimes we get multiple tokens at the same location:
317 //
318 // - findExplicitReferences() returns a heuristic result for a dependent name
319 // (e.g. Method) and CollectExtraHighlighting returning a fallback dependent
320 // highlighting (e.g. Unknown+Dependent).
321 // - macro arguments are expanded multiple times and have different roles
322 // - broken code recovery produces several AST nodes at the same location
323 //
324 // We should either resolve these to a single token, or drop them all.
325 // Our heuristics are:
326 //
327 // - token kinds that come with "dependent-name" modifiers are less reliable
328 // (these tend to be vague, like Type or Unknown)
329 // - if we have multiple equally reliable kinds, drop token rather than guess
330 // - take the union of modifiers from all tokens
331 //
332 // In particular, heuristically resolved dependent names get their heuristic
333 // kind, plus the dependent modifier.
334 std::optional<HighlightingToken> resolveConflict(const HighlightingToken &A,
335 const HighlightingToken &B) {
336 unsigned Priority1 = evaluateHighlightPriority(A);
337 unsigned Priority2 = evaluateHighlightPriority(B);
338 if (Priority1 == Priority2 && A.Kind != B.Kind)
339 return std::nullopt;
340 auto Result = Priority1 > Priority2 ? A : B;
341 Result.Modifiers = A.Modifiers | B.Modifiers;
342 return Result;
343 }
344 std::optional<HighlightingToken>
345 resolveConflict(ArrayRef<HighlightingToken> Tokens) {
346 if (Tokens.size() == 1)
347 return Tokens[0];
348
349 assert(Tokens.size() >= 2);
350 std::optional<HighlightingToken> Winner =
351 resolveConflict(Tokens[0], Tokens[1]);
352 for (size_t I = 2; Winner && I < Tokens.size(); ++I)
353 Winner = resolveConflict(*Winner, Tokens[I]);
354 return Winner;
355 }
356
357 /// Consumes source locations and maps them to text ranges for highlightings.
358 class HighlightingsBuilder {
359 public:
360 HighlightingsBuilder(const ParsedAST &AST)
361 : TB(AST.getTokens()), SourceMgr(AST.getSourceManager()),
362 LangOpts(AST.getLangOpts()) {}
363
364 HighlightingToken &addToken(SourceLocation Loc, HighlightingKind Kind) {
365 auto Range = getRangeForSourceLocation(Loc);
366 if (!Range)
367 return InvalidHighlightingToken;
368
369 return addToken(*Range, Kind);
370 }
371
372 HighlightingToken &addToken(Range R, HighlightingKind Kind) {
373 HighlightingToken HT;
374 HT.R = std::move(R);
375 HT.Kind = Kind;
376 Tokens.push_back(std::move(HT));
377 return Tokens.back();
378 }
379
380 void addExtraModifier(SourceLocation Loc, HighlightingModifier Modifier) {
381 if (auto Range = getRangeForSourceLocation(Loc))
382 ExtraModifiers[*Range].push_back(Modifier);
383 }
384
385 std::vector<HighlightingToken> collect(ParsedAST &AST) && {
386 // Initializer lists can give duplicates of tokens, therefore all tokens
387 // must be deduplicated.
388 llvm::sort(Tokens);
389 auto Last = std::unique(Tokens.begin(), Tokens.end());
390 Tokens.erase(Last, Tokens.end());
391
392 // Macros can give tokens that have the same source range but conflicting
393 // kinds. In this case all tokens sharing this source range should be
394 // removed.
395 std::vector<HighlightingToken> NonConflicting;
396 NonConflicting.reserve(Tokens.size());
397 for (ArrayRef<HighlightingToken> TokRef = Tokens; !TokRef.empty();) {
398 ArrayRef<HighlightingToken> Conflicting =
399 TokRef.take_while([&](const HighlightingToken &T) {
400 // TokRef is guaranteed at least one element here because otherwise
401 // this predicate would never fire.
402 return T.R == TokRef.front().R;
403 });
404 if (auto Resolved = resolveConflict(Conflicting)) {
405 // Apply extra collected highlighting modifiers
406 auto Modifiers = ExtraModifiers.find(Resolved->R);
407 if (Modifiers != ExtraModifiers.end()) {
408 for (HighlightingModifier Mod : Modifiers->second) {
409 Resolved->addModifier(Mod);
410 }
411 }
412
413 NonConflicting.push_back(*Resolved);
414 }
415 // TokRef[Conflicting.size()] is the next token with a different range (or
416 // the end of the Tokens).
417 TokRef = TokRef.drop_front(Conflicting.size());
418 }
419
420 const auto &SM = AST.getSourceManager();
421 StringRef MainCode = SM.getBufferOrFake(SM.getMainFileID()).getBuffer();
422
423 // Merge token stream with "inactive line" markers.
424 std::vector<HighlightingToken> WithInactiveLines;
425 auto SortedSkippedRanges = AST.getMacros().SkippedRanges;
426 llvm::sort(SortedSkippedRanges);
427 auto It = NonConflicting.begin();
428 for (const Range &R : SortedSkippedRanges) {
429 // Create one token for each line in the skipped range, so it works
430 // with line-based diffing.
431 assert(R.start.line <= R.end.line);
432 for (int Line = R.start.line; Line <= R.end.line; ++Line) {
433 // If the end of the inactive range is at the beginning
434 // of a line, that line is not inactive.
435 if (Line == R.end.line && R.end.character == 0)
436 continue;
437 // Copy tokens before the inactive line
438 for (; It != NonConflicting.end() && It->R.start.line < Line; ++It)
439 WithInactiveLines.push_back(std::move(*It));
440 // Add a token for the inactive line itself.
441 auto StartOfLine = positionToOffset(MainCode, Position{Line, 0});
442 if (StartOfLine) {
443 StringRef LineText =
444 MainCode.drop_front(*StartOfLine).take_until([](char C) {
445 return C == '\n';
446 });
447 HighlightingToken HT;
448 WithInactiveLines.emplace_back();
449 WithInactiveLines.back().Kind = HighlightingKind::InactiveCode;
450 WithInactiveLines.back().R.start.line = Line;
451 WithInactiveLines.back().R.end.line = Line;
452 WithInactiveLines.back().R.end.character =
453 static_cast<int>(lspLength(LineText));
454 } else {
455 elog("Failed to convert position to offset: {0}",
456 StartOfLine.takeError());
457 }
458
459 // Skip any other tokens on the inactive line. e.g.
460 // `#ifndef Foo` is considered as part of an inactive region when Foo is
461 // defined, and there is a Foo macro token.
462 // FIXME: we should reduce the scope of the inactive region to not
463 // include the directive itself.
464 while (It != NonConflicting.end() && It->R.start.line == Line)
465 ++It;
466 }
467 }
468 // Copy tokens after the last inactive line
469 for (; It != NonConflicting.end(); ++It)
470 WithInactiveLines.push_back(std::move(*It));
471 return WithInactiveLines;
472 }
473
474 const HeuristicResolver *getResolver() const { return Resolver; }
475
476 private:
477 std::optional<Range> getRangeForSourceLocation(SourceLocation Loc) {
478 Loc = getHighlightableSpellingToken(Loc, SourceMgr);
479 if (Loc.isInvalid())
480 return std::nullopt;
481
482 const auto *Tok = TB.spelledTokenAt(Loc);
483 assert(Tok);
484
485 return halfOpenToRange(SourceMgr,
486 Tok->range(SourceMgr).toCharRange(SourceMgr));
487 }
488
489 const syntax::TokenBuffer &TB;
490 const SourceManager &SourceMgr;
491 const LangOptions &LangOpts;
492 std::vector<HighlightingToken> Tokens;
493 std::map<Range, llvm::SmallVector<HighlightingModifier, 1>> ExtraModifiers;
494 const HeuristicResolver *Resolver = nullptr;
495 // returned from addToken(InvalidLoc)
496 HighlightingToken InvalidHighlightingToken;
497 };
498
499 std::optional<HighlightingModifier> scopeModifier(const NamedDecl *D) {
500 const DeclContext *DC = D->getDeclContext();
501 // Injected "Foo" within the class "Foo" has file scope, not class scope.
502 if (auto *R = dyn_cast_or_null<RecordDecl>(D))
503 if (R->isInjectedClassName())
504 DC = DC->getParent();
505 // Lambda captures are considered function scope, not class scope.
506 if (llvm::isa<FieldDecl>(D))
507 if (const auto *RD = llvm::dyn_cast<RecordDecl>(DC))
508 if (RD->isLambda())
509 return HighlightingModifier::FunctionScope;
510 // Walk up the DeclContext hierarchy until we find something interesting.
511 for (; !DC->isFileContext(); DC = DC->getParent()) {
512 if (DC->isFunctionOrMethod())
513 return HighlightingModifier::FunctionScope;
514 if (DC->isRecord())
515 return HighlightingModifier::ClassScope;
516 }
517 // Some template parameters (e.g. those for variable templates) don't have
518 // meaningful DeclContexts. That doesn't mean they're global!
519 if (DC->isTranslationUnit() && D->isTemplateParameter())
520 return std::nullopt;
521 // ExternalLinkage threshold could be tweaked, e.g. module-visible as global.
522 if (D->getLinkageInternal() < ExternalLinkage)
523 return HighlightingModifier::FileScope;
524 return HighlightingModifier::GlobalScope;
525 }
526
527 std::optional<HighlightingModifier> scopeModifier(const Type *T) {
528 if (!T)
529 return std::nullopt;
530 if (T->isBuiltinType())
531 return HighlightingModifier::GlobalScope;
532 if (auto *TD = dyn_cast<TemplateTypeParmType>(T))
533 return scopeModifier(TD->getDecl());
534 if (auto *TD = T->getAsTagDecl())
535 return scopeModifier(TD);
536 return std::nullopt;
537 }
538
539 /// Produces highlightings, which are not captured by findExplicitReferences,
540 /// e.g. highlights dependent names and 'auto' as the underlying type.
541 class CollectExtraHighlightings
542 : public RecursiveASTVisitor<CollectExtraHighlightings> {
543 using Base = RecursiveASTVisitor<CollectExtraHighlightings>;
544
545 public:
546 CollectExtraHighlightings(HighlightingsBuilder &H) : H(H) {}
547
548 bool VisitCXXConstructExpr(CXXConstructExpr *E) {
549 highlightMutableReferenceArguments(E->getConstructor(),
550 {E->getArgs(), E->getNumArgs()});
551
552 return true;
553 }
554
555 bool TraverseConstructorInitializer(CXXCtorInitializer *Init) {
556 if (Init->isMemberInitializer())
557 if (auto *Member = Init->getMember())
558 highlightMutableReferenceArgument(Member->getType(), Init->getInit());
559 return Base::TraverseConstructorInitializer(Init);
560 }
561
562 bool VisitPredefinedExpr(PredefinedExpr *E) {
563 H.addToken(E->getLocation(), HighlightingKind::LocalVariable)
564 .addModifier(HighlightingModifier::Static)
565 .addModifier(HighlightingModifier::Readonly)
566 .addModifier(HighlightingModifier::FunctionScope);
567 return true;
568 }
569
570 bool VisitFunctionDecl(FunctionDecl *D) {
571 if (D->isOverloadedOperator()) {
572 const auto AddOpDeclToken = [&](SourceLocation Loc) {
573 auto &Token = H.addToken(Loc, HighlightingKind::Operator)
574 .addModifier(HighlightingModifier::Declaration);
575 if (D->isThisDeclarationADefinition())
576 Token.addModifier(HighlightingModifier::Definition);
577 };
578 const auto Range = D->getNameInfo().getCXXOperatorNameRange();
579 AddOpDeclToken(Range.getBegin());
580 const auto Kind = D->getOverloadedOperator();
581 if (Kind == OO_Call || Kind == OO_Subscript)
582 AddOpDeclToken(Range.getEnd());
583 }
584 return true;
585 }
586
587 bool VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
588 const auto AddOpToken = [&](SourceLocation Loc) {
589 H.addToken(Loc, HighlightingKind::Operator)
590 .addModifier(HighlightingModifier::UserDefined);
591 };
592 AddOpToken(E->getOperatorLoc());
593 const auto Kind = E->getOperator();
594 if (Kind == OO_Call || Kind == OO_Subscript) {
595 if (auto *Callee = E->getCallee())
596 AddOpToken(Callee->getBeginLoc());
597 }
598 return true;
599 }
600
601 bool VisitUnaryOperator(UnaryOperator *Op) {
602 auto &Token = H.addToken(Op->getOperatorLoc(), HighlightingKind::Operator);
603 if (Op->getSubExpr()->isTypeDependent())
604 Token.addModifier(HighlightingModifier::UserDefined);
605 return true;
606 }
607
608 bool VisitBinaryOperator(BinaryOperator *Op) {
609 auto &Token = H.addToken(Op->getOperatorLoc(), HighlightingKind::Operator);
610 if (Op->getLHS()->isTypeDependent() || Op->getRHS()->isTypeDependent())
611 Token.addModifier(HighlightingModifier::UserDefined);
612 return true;
613 }
614
615 bool VisitConditionalOperator(ConditionalOperator *Op) {
616 H.addToken(Op->getQuestionLoc(), HighlightingKind::Operator);
617 H.addToken(Op->getColonLoc(), HighlightingKind::Operator);
618 return true;
619 }
620
621 bool VisitCXXNewExpr(CXXNewExpr *E) {
622 auto &Token = H.addToken(E->getBeginLoc(), HighlightingKind::Operator);
623 if (isa_and_present<CXXMethodDecl>(E->getOperatorNew()))
624 Token.addModifier(HighlightingModifier::UserDefined);
625 return true;
626 }
627
628 bool VisitCXXDeleteExpr(CXXDeleteExpr *E) {
629 auto &Token = H.addToken(E->getBeginLoc(), HighlightingKind::Operator);
630 if (isa_and_present<CXXMethodDecl>(E->getOperatorDelete()))
631 Token.addModifier(HighlightingModifier::UserDefined);
632 return true;
633 }
634
635 bool VisitCallExpr(CallExpr *E) {
636 // Highlighting parameters passed by non-const reference does not really
637 // make sense for literals...
638 if (isa<UserDefinedLiteral>(E))
639 return true;
640
641 // FIXME: consider highlighting parameters of some other overloaded
642 // operators as well
643 llvm::ArrayRef<const Expr *> Args = {E->getArgs(), E->getNumArgs()};
644 if (auto *CallOp = dyn_cast<CXXOperatorCallExpr>(E)) {
645 switch (CallOp->getOperator()) {
646 case OO_Call:
647 case OO_Subscript:
648 Args = Args.drop_front(); // Drop object parameter
649 break;
650 default:
651 return true;
652 }
653 }
654
655 highlightMutableReferenceArguments(
656 dyn_cast_or_null<FunctionDecl>(E->getCalleeDecl()), Args);
657
658 return true;
659 }
660
661 void highlightMutableReferenceArgument(QualType T, const Expr *Arg) {
662 if (!Arg)
663 return;
664
665 // Is this parameter passed by non-const pointer or reference?
666 // FIXME The condition T->idDependentType() could be relaxed a bit,
667 // e.g. std::vector<T>& is dependent but we would want to highlight it
668 bool IsRef = T->isLValueReferenceType();
669 bool IsPtr = T->isPointerType();
670 if ((!IsRef && !IsPtr) || T->getPointeeType().isConstQualified() ||
671 T->isDependentType()) {
672 return;
673 }
674
675 std::optional<SourceLocation> Location;
676
677 // FIXME Add "unwrapping" for ArraySubscriptExpr,
678 // e.g. highlight `a` in `a[i]`
679 // FIXME Handle dependent expression types
680 if (auto *IC = dyn_cast<ImplicitCastExpr>(Arg))
681 Arg = IC->getSubExprAsWritten();
682 if (auto *UO = dyn_cast<UnaryOperator>(Arg)) {
683 if (UO->getOpcode() == UO_AddrOf)
684 Arg = UO->getSubExpr();
685 }
686 if (auto *DR = dyn_cast<DeclRefExpr>(Arg))
687 Location = DR->getLocation();
688 else if (auto *M = dyn_cast<MemberExpr>(Arg))
689 Location = M->getMemberLoc();
690
691 if (Location)
692 H.addExtraModifier(*Location,
693 IsRef ? HighlightingModifier::UsedAsMutableReference
694 : HighlightingModifier::UsedAsMutablePointer);
695 }
696
697 void
698 highlightMutableReferenceArguments(const FunctionDecl *FD,
699 llvm::ArrayRef<const Expr *const> Args) {
700 if (!FD)
701 return;
702
703 if (auto *ProtoType = FD->getType()->getAs<FunctionProtoType>()) {
704 // Iterate over the types of the function parameters.
705 // If any of them are non-const reference paramteres, add it as a
706 // highlighting modifier to the corresponding expression
707 for (size_t I = 0;
708 I < std::min(size_t(ProtoType->getNumParams()), Args.size()); ++I) {
709 highlightMutableReferenceArgument(ProtoType->getParamType(I), Args[I]);
710 }
711 }
712 }
713
714 bool VisitDecltypeTypeLoc(DecltypeTypeLoc L) {
715 if (auto K = kindForType(L.getTypePtr(), H.getResolver())) {
716 auto &Tok = H.addToken(L.getBeginLoc(), *K)
717 .addModifier(HighlightingModifier::Deduced);
718 if (auto Mod = scopeModifier(L.getTypePtr()))
719 Tok.addModifier(*Mod);
720 if (isDefaultLibrary(L.getTypePtr()))
721 Tok.addModifier(HighlightingModifier::DefaultLibrary);
722 }
723 return true;
724 }
725
726 bool VisitCXXDestructorDecl(CXXDestructorDecl *D) {
727 if (auto *TI = D->getNameInfo().getNamedTypeInfo()) {
728 SourceLocation Loc = TI->getTypeLoc().getBeginLoc();
729 H.addExtraModifier(Loc, HighlightingModifier::ConstructorOrDestructor);
730 H.addExtraModifier(Loc, HighlightingModifier::Declaration);
731 if (D->isThisDeclarationADefinition())
732 H.addExtraModifier(Loc, HighlightingModifier::Definition);
733 }
734 return true;
735 }
736
737 bool VisitCXXMemberCallExpr(CXXMemberCallExpr *CE) {
738 // getMethodDecl can return nullptr with member pointers, e.g.
739 // `(foo.*pointer_to_member_fun)(arg);`
740 if (auto *D = CE->getMethodDecl()) {
741 if (isa<CXXDestructorDecl>(D)) {
742 if (auto *ME = dyn_cast<MemberExpr>(CE->getCallee())) {
743 if (auto *TI = ME->getMemberNameInfo().getNamedTypeInfo()) {
744 H.addExtraModifier(TI->getTypeLoc().getBeginLoc(),
745 HighlightingModifier::ConstructorOrDestructor);
746 }
747 }
748 } else if (D->isOverloadedOperator()) {
749 if (auto *ME = dyn_cast<MemberExpr>(CE->getCallee()))
750 H.addToken(
751 ME->getMemberNameInfo().getCXXOperatorNameRange().getBegin(),
752 HighlightingKind::Operator)
753 .addModifier(HighlightingModifier::UserDefined);
754 }
755 }
756 return true;
757 }
758
759 bool VisitDeclaratorDecl(DeclaratorDecl *D) {
760 auto *AT = D->getType()->getContainedAutoType();
761 if (!AT)
762 return true;
763 auto K =
764 kindForType(AT->getDeducedType().getTypePtrOrNull(), H.getResolver());
765 if (!K)
766 return true;
767 SourceLocation StartLoc = D->getTypeSpecStartLoc();
768 // The AutoType may not have a corresponding token, e.g. in the case of
769 // init-captures. In this case, StartLoc overlaps with the location
770 // of the decl itself, and producing a token for the type here would result
771 // in both it and the token for the decl being dropped due to conflict.
772 if (StartLoc == D->getLocation())
773 return true;
774 auto &Tok =
775 H.addToken(StartLoc, *K).addModifier(HighlightingModifier::Deduced);
776 const Type *Deduced = AT->getDeducedType().getTypePtrOrNull();
777 if (auto Mod = scopeModifier(Deduced))
778 Tok.addModifier(*Mod);
779 if (isDefaultLibrary(Deduced))
780 Tok.addModifier(HighlightingModifier::DefaultLibrary);
781 return true;
782 }
783
784 // We handle objective-C selectors specially, because one reference can
785 // cover several non-contiguous tokens.
786 void highlightObjCSelector(const ArrayRef<SourceLocation> &Locs, bool Decl,
787 bool Def, bool Class, bool DefaultLibrary) {
788 HighlightingKind Kind =
789 Class ? HighlightingKind::StaticMethod : HighlightingKind::Method;
790 for (SourceLocation Part : Locs) {
791 auto &Tok =
792 H.addToken(Part, Kind).addModifier(HighlightingModifier::ClassScope);
793 if (Decl)
794 Tok.addModifier(HighlightingModifier::Declaration);
795 if (Def)
796 Tok.addModifier(HighlightingModifier::Definition);
797 if (Class)
798 Tok.addModifier(HighlightingModifier::Static);
799 if (DefaultLibrary)
800 Tok.addModifier(HighlightingModifier::DefaultLibrary);
801 }
802 }
803
804 bool VisitObjCMethodDecl(ObjCMethodDecl *OMD) {
805 llvm::SmallVector<SourceLocation> Locs;
806 OMD->getSelectorLocs(Locs);
807 highlightObjCSelector(Locs, /*Decl=*/true,
808 OMD->isThisDeclarationADefinition(),
809 OMD->isClassMethod(), isDefaultLibrary(OMD));
810 return true;
811 }
812
813 bool VisitObjCMessageExpr(ObjCMessageExpr *OME) {
814 llvm::SmallVector<SourceLocation> Locs;
815 OME->getSelectorLocs(Locs);
816 bool DefaultLibrary = false;
817 if (ObjCMethodDecl *OMD = OME->getMethodDecl())
818 DefaultLibrary = isDefaultLibrary(OMD);
819 highlightObjCSelector(Locs, /*Decl=*/false, /*Def=*/false,
820 OME->isClassMessage(), DefaultLibrary);
821 return true;
822 }
823
824 // Objective-C allows you to use property syntax `self.prop` as sugar for
825 // `[self prop]` and `[self setProp:]` when there's no explicit `@property`
826 // for `prop` as well as for class properties. We treat this like a property
827 // even though semantically it's equivalent to a method expression.
828 void highlightObjCImplicitPropertyRef(const ObjCMethodDecl *OMD,
829 SourceLocation Loc) {
830 auto &Tok = H.addToken(Loc, HighlightingKind::Field)
831 .addModifier(HighlightingModifier::ClassScope);
832 if (OMD->isClassMethod())
833 Tok.addModifier(HighlightingModifier::Static);
834 if (isDefaultLibrary(OMD))
835 Tok.addModifier(HighlightingModifier::DefaultLibrary);
836 }
837
838 bool VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *OPRE) {
839 // We need to handle implicit properties here since they will appear to
840 // reference `ObjCMethodDecl` via an implicit `ObjCMessageExpr`, so normal
841 // highlighting will not work.
842 if (!OPRE->isImplicitProperty())
843 return true;
844 // A single property expr can reference both a getter and setter, but we can
845 // only provide a single semantic token, so prefer the getter. In most cases
846 // the end result should be the same, although it's technically possible
847 // that the user defines a setter for a system SDK.
848 if (OPRE->isMessagingGetter()) {
849 highlightObjCImplicitPropertyRef(OPRE->getImplicitPropertyGetter(),
850 OPRE->getLocation());
851 return true;
852 }
853 if (OPRE->isMessagingSetter()) {
854 highlightObjCImplicitPropertyRef(OPRE->getImplicitPropertySetter(),
855 OPRE->getLocation());
856 }
857 return true;
858 }
859
860 bool VisitOverloadExpr(OverloadExpr *E) {
861 if (!E->decls().empty())
862 return true; // handled by findExplicitReferences.
863 auto &Tok = H.addToken(E->getNameLoc(), HighlightingKind::Unknown)
864 .addModifier(HighlightingModifier::DependentName);
865 if (llvm::isa<UnresolvedMemberExpr>(E))
866 Tok.addModifier(HighlightingModifier::ClassScope);
867 // other case is UnresolvedLookupExpr, scope is unknown.
868 return true;
869 }
870
871 bool VisitCXXDependentScopeMemberExpr(CXXDependentScopeMemberExpr *E) {
872 H.addToken(E->getMemberNameInfo().getLoc(), HighlightingKind::Unknown)
873 .addModifier(HighlightingModifier::DependentName)
874 .addModifier(HighlightingModifier::ClassScope);
875 return true;
876 }
877
878 bool VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) {
879 H.addToken(E->getNameInfo().getLoc(), HighlightingKind::Unknown)
880 .addModifier(HighlightingModifier::DependentName)
881 .addModifier(HighlightingModifier::ClassScope);
882 return true;
883 }
884
885 bool VisitAttr(Attr *A) {
886 switch (A->getKind()) {
887 case attr::Override:
888 case attr::Final:
889 H.addToken(A->getLocation(), HighlightingKind::Modifier);
890 break;
891 default:
892 break;
893 }
894 return true;
895 }
896
897 bool VisitDependentNameTypeLoc(DependentNameTypeLoc L) {
898 H.addToken(L.getNameLoc(), HighlightingKind::Type)
899 .addModifier(HighlightingModifier::DependentName)
900 .addModifier(HighlightingModifier::ClassScope);
901 return true;
902 }
903
904 bool VisitDependentTemplateSpecializationTypeLoc(
905 DependentTemplateSpecializationTypeLoc L) {
906 H.addToken(L.getTemplateNameLoc(), HighlightingKind::Type)
907 .addModifier(HighlightingModifier::DependentName)
908 .addModifier(HighlightingModifier::ClassScope);
909 return true;
910 }
911
912 bool TraverseTemplateArgumentLoc(TemplateArgumentLoc L) {
913 // Handle template template arguments only (other arguments are handled by
914 // their Expr, TypeLoc etc values).
915 if (L.getArgument().getKind() != TemplateArgument::Template &&
916 L.getArgument().getKind() != TemplateArgument::TemplateExpansion)
917 return RecursiveASTVisitor::TraverseTemplateArgumentLoc(L);
918
919 TemplateName N = L.getArgument().getAsTemplateOrTemplatePattern();
920 switch (N.getKind()) {
921 case TemplateName::OverloadedTemplate:
922 // Template template params must always be class templates.
923 // Don't bother to try to work out the scope here.
924 H.addToken(L.getTemplateNameLoc(), HighlightingKind::Class);
925 break;
926 case TemplateName::DependentTemplate:
927 case TemplateName::AssumedTemplate:
928 H.addToken(L.getTemplateNameLoc(), HighlightingKind::Class)
929 .addModifier(HighlightingModifier::DependentName);
930 break;
931 case TemplateName::Template:
932 case TemplateName::QualifiedTemplate:
933 case TemplateName::SubstTemplateTemplateParm:
934 case TemplateName::SubstTemplateTemplateParmPack:
935 case TemplateName::UsingTemplate:
936 // Names that could be resolved to a TemplateDecl are handled elsewhere.
937 break;
938 }
939 return RecursiveASTVisitor::TraverseTemplateArgumentLoc(L);
940 }
941
942 // findExplicitReferences will walk nested-name-specifiers and
943 // find anything that can be resolved to a Decl. However, non-leaf
944 // components of nested-name-specifiers which are dependent names
945 // (kind "Identifier") cannot be resolved to a decl, so we visit
946 // them here.
947 bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc Q) {
948 if (NestedNameSpecifier *NNS = Q.getNestedNameSpecifier()) {
949 if (NNS->getKind() == NestedNameSpecifier::Identifier)
950 H.addToken(Q.getLocalBeginLoc(), HighlightingKind::Type)
951 .addModifier(HighlightingModifier::DependentName)
952 .addModifier(HighlightingModifier::ClassScope);
953 }
954 return RecursiveASTVisitor::TraverseNestedNameSpecifierLoc(Q);
955 }
956
957 private:
958 HighlightingsBuilder &H;
959 };
960 } // namespace
961
962 std::vector<HighlightingToken> getSemanticHighlightings(ParsedAST &AST) {
963 auto &C = AST.getASTContext();
964 // Add highlightings for AST nodes.
965 HighlightingsBuilder Builder(AST);
966 // Highlight 'decltype' and 'auto' as their underlying types.
967 CollectExtraHighlightings(Builder).TraverseAST(C);
968 // Highlight all decls and references coming from the AST.
969 findExplicitReferences(
970 C,
971 [&](ReferenceLoc R) {
972 for (const NamedDecl *Decl : R.Targets) {
973 if (!canHighlightName(Decl->getDeclName()))
974 continue;
975 auto Kind = kindForDecl(Decl, AST.getHeuristicResolver());
976 if (!Kind)
977 continue;
978 auto &Tok = Builder.addToken(R.NameLoc, *Kind);
979
980 // The attribute tests don't want to look at the template.
981 if (auto *TD = dyn_cast<TemplateDecl>(Decl)) {
982 if (auto *Templated = TD->getTemplatedDecl())
983 Decl = Templated;
984 }
985 if (auto Mod = scopeModifier(Decl))
986 Tok.addModifier(*Mod);
987 if (isConst(Decl))
988 Tok.addModifier(HighlightingModifier::Readonly);
989 if (isStatic(Decl))
990 Tok.addModifier(HighlightingModifier::Static);
991 if (isAbstract(Decl))
992 Tok.addModifier(HighlightingModifier::Abstract);
993 if (isVirtual(Decl))
994 Tok.addModifier(HighlightingModifier::Virtual);
995 if (isDependent(Decl))
996 Tok.addModifier(HighlightingModifier::DependentName);
997 if (isDefaultLibrary(Decl))
998 Tok.addModifier(HighlightingModifier::DefaultLibrary);
999 if (Decl->isDeprecated())
1000 Tok.addModifier(HighlightingModifier::Deprecated);
1001 if (isa<CXXConstructorDecl>(Decl))
1002 Tok.addModifier(HighlightingModifier::ConstructorOrDestructor);
1003 if (R.IsDecl) {
1004 // Do not treat an UnresolvedUsingValueDecl as a declaration.
1005 // It's more common to think of it as a reference to the
1006 // underlying declaration.
1007 if (!isa<UnresolvedUsingValueDecl>(Decl))
1008 Tok.addModifier(HighlightingModifier::Declaration);
1009 if (isUniqueDefinition(Decl))
1010 Tok.addModifier(HighlightingModifier::Definition);
1011 }
1012 }
1013 },
1014 AST.getHeuristicResolver());
1015 // Add highlightings for macro references.
1016 auto AddMacro = [&](const MacroOccurrence &M) {
1017 auto &T = Builder.addToken(M.Rng, HighlightingKind::Macro);
1018 T.addModifier(HighlightingModifier::GlobalScope);
1019 if (M.IsDefinition)
1020 T.addModifier(HighlightingModifier::Declaration);
1021 };
1022 for (const auto &SIDToRefs : AST.getMacros().MacroRefs)
1023 for (const auto &M : SIDToRefs.second)
1024 AddMacro(M);
1025 for (const auto &M : AST.getMacros().UnknownMacros)
1026 AddMacro(M);
1027
1028 return std::move(Builder).collect(AST);
1029 }
1030
1031 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingKind K) {
1032 switch (K) {
1033 case HighlightingKind::Variable:
1034 return OS << "Variable";
1035 case HighlightingKind::LocalVariable:
1036 return OS << "LocalVariable";
1037 case HighlightingKind::Parameter:
1038 return OS << "Parameter";
1039 case HighlightingKind::Function:
1040 return OS << "Function";
1041 case HighlightingKind::Method:
1042 return OS << "Method";
1043 case HighlightingKind::StaticMethod:
1044 return OS << "StaticMethod";
1045 case HighlightingKind::Field:
1046 return OS << "Field";
1047 case HighlightingKind::StaticField:
1048 return OS << "StaticField";
1049 case HighlightingKind::Class:
1050 return OS << "Class";
1051 case HighlightingKind::Interface:
1052 return OS << "Interface";
1053 case HighlightingKind::Enum:
1054 return OS << "Enum";
1055 case HighlightingKind::EnumConstant:
1056 return OS << "EnumConstant";
1057 case HighlightingKind::Typedef:
1058 return OS << "Typedef";
1059 case HighlightingKind::Type:
1060 return OS << "Type";
1061 case HighlightingKind::Unknown:
1062 return OS << "Unknown";
1063 case HighlightingKind::Namespace:
1064 return OS << "Namespace";
1065 case HighlightingKind::TemplateParameter:
1066 return OS << "TemplateParameter";
1067 case HighlightingKind::Concept:
1068 return OS << "Concept";
1069 case HighlightingKind::Primitive:
1070 return OS << "Primitive";
1071 case HighlightingKind::Macro:
1072 return OS << "Macro";
1073 case HighlightingKind::Modifier:
1074 return OS << "Modifier";
1075 case HighlightingKind::Operator:
1076 return OS << "Operator";
1077 case HighlightingKind::InactiveCode:
1078 return OS << "InactiveCode";
1079 }
1080 llvm_unreachable("invalid HighlightingKind");
1081 }
1082 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingModifier K) {
1083 switch (K) {
1084 case HighlightingModifier::Declaration:
1085 return OS << "decl"; // abbreviation for common case
1086 case HighlightingModifier::Definition:
1087 return OS << "def"; // abbrevation for common case
1088 case HighlightingModifier::ConstructorOrDestructor:
1089 return OS << "constrDestr";
1090 default:
1091 return OS << toSemanticTokenModifier(K);
1092 }
1093 }
1094
1095 bool operator==(const HighlightingToken &L, const HighlightingToken &R) {
1096 return std::tie(L.R, L.Kind, L.Modifiers) ==
1097 std::tie(R.R, R.Kind, R.Modifiers);
1098 }
1099 bool operator<(const HighlightingToken &L, const HighlightingToken &R) {
1100 return std::tie(L.R, L.Kind, L.Modifiers) <
1101 std::tie(R.R, R.Kind, R.Modifiers);
1102 }
1103
1104 std::vector<SemanticToken>
1105 toSemanticTokens(llvm::ArrayRef<HighlightingToken> Tokens,
1106 llvm::StringRef Code) {
1107 assert(llvm::is_sorted(Tokens));
1108 std::vector<SemanticToken> Result;
1109 // In case we split a HighlightingToken into multiple tokens (e.g. because it
1110 // was spanning multiple lines), this tracks the last one. This prevents
1111 // having a copy all the time.
1112 HighlightingToken Scratch;
1113 const HighlightingToken *Last = nullptr;
1114 for (const HighlightingToken &Tok : Tokens) {
1115 Result.emplace_back();
1116 SemanticToken *Out = &Result.back();
1117 // deltaStart/deltaLine are relative if possible.
1118 if (Last) {
1119 assert(Tok.R.start.line >= Last->R.end.line);
1120 Out->deltaLine = Tok.R.start.line - Last->R.end.line;
1121 if (Out->deltaLine == 0) {
1122 assert(Tok.R.start.character >= Last->R.start.character);
1123 Out->deltaStart = Tok.R.start.character - Last->R.start.character;
1124 } else {
1125 Out->deltaStart = Tok.R.start.character;
1126 }
1127 } else {
1128 Out->deltaLine = Tok.R.start.line;
1129 Out->deltaStart = Tok.R.start.character;
1130 }
1131 Out->tokenType = static_cast<unsigned>(Tok.Kind);
1132 Out->tokenModifiers = Tok.Modifiers;
1133 Last = &Tok;
1134
1135 if (Tok.R.end.line == Tok.R.start.line) {
1136 Out->length = Tok.R.end.character - Tok.R.start.character;
1137 } else {
1138 // If the token spans a line break, split it into multiple pieces for each
1139 // line.
1140 // This is slow, but multiline tokens are rare.
1141 // FIXME: There's a client capability for supporting multiline tokens,
1142 // respect that.
1143 auto TokStartOffset = llvm::cantFail(positionToOffset(Code, Tok.R.start));
1144 // Note that the loop doesn't cover the last line, which has a special
1145 // length.
1146 for (int I = Tok.R.start.line; I < Tok.R.end.line; ++I) {
1147 auto LineEnd = Code.find('\n', TokStartOffset);
1148 assert(LineEnd != Code.npos);
1149 Out->length = LineEnd - TokStartOffset;
1150 // Token continues on next line, right after the line break.
1151 TokStartOffset = LineEnd + 1;
1152 Result.emplace_back();
1153 Out = &Result.back();
1154 *Out = Result[Result.size() - 2];
1155 // New token starts at the first column of the next line.
1156 Out->deltaLine = 1;
1157 Out->deltaStart = 0;
1158 }
1159 // This is the token on last line.
1160 Out->length = Tok.R.end.character;
1161 // Update the start location for last token, as that's used in the
1162 // relative delta calculation for following tokens.
1163 Scratch = *Last;
1164 Scratch.R.start.line = Tok.R.end.line;
1165 Scratch.R.start.character = 0;
1166 Last = &Scratch;
1167 }
1168 }
1169 return Result;
1170 }
1171 llvm::StringRef toSemanticTokenType(HighlightingKind Kind) {
1172 switch (Kind) {
1173 case HighlightingKind::Variable:
1174 case HighlightingKind::LocalVariable:
1175 case HighlightingKind::StaticField:
1176 return "variable";
1177 case HighlightingKind::Parameter:
1178 return "parameter";
1179 case HighlightingKind::Function:
1180 return "function";
1181 case HighlightingKind::Method:
1182 return "method";
1183 case HighlightingKind::StaticMethod:
1184 // FIXME: better method with static modifier?
1185 return "function";
1186 case HighlightingKind::Field:
1187 return "property";
1188 case HighlightingKind::Class:
1189 return "class";
1190 case HighlightingKind::Interface:
1191 return "interface";
1192 case HighlightingKind::Enum:
1193 return "enum";
1194 case HighlightingKind::EnumConstant:
1195 return "enumMember";
1196 case HighlightingKind::Typedef:
1197 case HighlightingKind::Type:
1198 return "type";
1199 case HighlightingKind::Unknown:
1200 return "unknown"; // nonstandard
1201 case HighlightingKind::Namespace:
1202 return "namespace";
1203 case HighlightingKind::TemplateParameter:
1204 return "typeParameter";
1205 case HighlightingKind::Concept:
1206 return "concept"; // nonstandard
1207 case HighlightingKind::Primitive:
1208 return "type";
1209 case HighlightingKind::Macro:
1210 return "macro";
1211 case HighlightingKind::Modifier:
1212 return "modifier";
1213 case HighlightingKind::Operator:
1214 return "operator";
1215 case HighlightingKind::InactiveCode:
1216 return "comment";
1217 }
1218 llvm_unreachable("unhandled HighlightingKind");
1219 }
1220
1221 llvm::StringRef toSemanticTokenModifier(HighlightingModifier Modifier) {
1222 switch (Modifier) {
1223 case HighlightingModifier::Declaration:
1224 return "declaration";
1225 case HighlightingModifier::Definition:
1226 return "definition";
1227 case HighlightingModifier::Deprecated:
1228 return "deprecated";
1229 case HighlightingModifier::Readonly:
1230 return "readonly";
1231 case HighlightingModifier::Static:
1232 return "static";
1233 case HighlightingModifier::Deduced:
1234 return "deduced"; // nonstandard
1235 case HighlightingModifier::Abstract:
1236 return "abstract";
1237 case HighlightingModifier::Virtual:
1238 return "virtual";
1239 case HighlightingModifier::DependentName:
1240 return "dependentName"; // nonstandard
1241 case HighlightingModifier::DefaultLibrary:
1242 return "defaultLibrary";
1243 case HighlightingModifier::UsedAsMutableReference:
1244 return "usedAsMutableReference"; // nonstandard
1245 case HighlightingModifier::UsedAsMutablePointer:
1246 return "usedAsMutablePointer"; // nonstandard
1247 case HighlightingModifier::ConstructorOrDestructor:
1248 return "constructorOrDestructor"; // nonstandard
1249 case HighlightingModifier::UserDefined:
1250 return "userDefined"; // nonstandard
1251 case HighlightingModifier::FunctionScope:
1252 return "functionScope"; // nonstandard
1253 case HighlightingModifier::ClassScope:
1254 return "classScope"; // nonstandard
1255 case HighlightingModifier::FileScope:
1256 return "fileScope"; // nonstandard
1257 case HighlightingModifier::GlobalScope:
1258 return "globalScope"; // nonstandard
1259 }
1260 llvm_unreachable("unhandled HighlightingModifier");
1261 }
1262
1263 std::vector<SemanticTokensEdit>
1264 diffTokens(llvm::ArrayRef<SemanticToken> Old,
1265 llvm::ArrayRef<SemanticToken> New) {
1266 // For now, just replace everything from the first-last modification.
1267 // FIXME: use a real diff instead, this is bad with include-insertion.
1268
1269 unsigned Offset = 0;
1270 while (!Old.empty() && !New.empty() && Old.front() == New.front()) {
1271 ++Offset;
1272 Old = Old.drop_front();
1273 New = New.drop_front();
1274 }
1275 while (!Old.empty() && !New.empty() && Old.back() == New.back()) {
1276 Old = Old.drop_back();
1277 New = New.drop_back();
1278 }
1279
1280 if (Old.empty() && New.empty())
1281 return {};
1282 SemanticTokensEdit Edit;
1283 Edit.startToken = Offset;
1284 Edit.deleteTokens = Old.size();
1285 Edit.tokens = New;
1286 return {std::move(Edit)};
1287 }
1288
1289 } // namespace clangd
1290 } // namespace clang