//==- SemanticHighlightingTests.cpp - SemanticHighlighting tests-*- C++ -* -==// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "Annotations.h" #include "Protocol.h" #include "SemanticHighlighting.h" #include "SourceCode.h" #include "TestTU.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/Error.h" #include "llvm/Support/ScopedPrinter.h" #include "llvm/Support/raw_ostream.h" #include "gmock/gmock.h" #include namespace clang { namespace clangd { namespace { using testing::IsEmpty; using testing::SizeIs; /// Annotates the input code with provided semantic highlightings. Results look /// something like: /// class $Class[[X]] { /// $Primitive[[int]] $Field[[a]] = 0; /// }; std::string annotate(llvm::StringRef Input, llvm::ArrayRef Tokens) { assert(llvm::is_sorted( Tokens, [](const HighlightingToken &L, const HighlightingToken &R) { return L.R.start < R.R.start; })); std::string Buf; llvm::raw_string_ostream OS(Buf); unsigned NextChar = 0; for (auto &T : Tokens) { unsigned StartOffset = llvm::cantFail(positionToOffset(Input, T.R.start)); unsigned EndOffset = llvm::cantFail(positionToOffset(Input, T.R.end)); assert(StartOffset <= EndOffset); assert(NextChar <= StartOffset); bool hasDef = T.Modifiers & (1 << uint32_t(HighlightingModifier::Definition)); bool hasDecl = T.Modifiers & (1 << uint32_t(HighlightingModifier::Declaration)); EXPECT_TRUE(!hasDef || hasDecl); OS << Input.substr(NextChar, StartOffset - NextChar); OS << '$' << T.Kind; for (unsigned I = 0; I <= static_cast(HighlightingModifier::LastModifier); ++I) { if (T.Modifiers & (1 << I)) { // _decl_def is common and redundant, just print _def instead. if (I != uint32_t(HighlightingModifier::Declaration) || !hasDef) OS << '_' << static_cast(I); } } OS << "[[" << Input.substr(StartOffset, EndOffset - StartOffset) << "]]"; NextChar = EndOffset; } OS << Input.substr(NextChar); return std::move(OS.str()); } void checkHighlightings(llvm::StringRef Code, std::vector> AdditionalFiles = {}, uint32_t ModifierMask = -1, std::vector AdditionalArgs = {}) { Annotations Test(Code); TestTU TU; TU.Code = std::string(Test.code()); TU.ExtraArgs.push_back("-std=c++20"); TU.ExtraArgs.push_back("-xobjective-c++"); TU.ExtraArgs.insert(std::end(TU.ExtraArgs), std::begin(AdditionalArgs), std::end(AdditionalArgs)); for (auto File : AdditionalFiles) TU.AdditionalFiles.insert({File.first, std::string(File.second)}); auto AST = TU.build(); auto Actual = getSemanticHighlightings(AST); for (auto &Token : Actual) Token.Modifiers &= ModifierMask; EXPECT_EQ(Code, annotate(Test.code(), Actual)); } constexpr static uint32_t ScopeModifierMask = 1 << unsigned(HighlightingModifier::FunctionScope) | 1 << unsigned(HighlightingModifier::ClassScope) | 1 << unsigned(HighlightingModifier::FileScope) | 1 << unsigned(HighlightingModifier::GlobalScope); TEST(SemanticHighlighting, GetsCorrectTokens) { const char *TestCases[] = { R"cpp( struct $Class_def[[AS]] { double $Field_decl[[SomeMember]]; }; struct { } $Variable_def[[S]]; void $Function_def[[foo]](int $Parameter_def[[A]], $Class[[AS]] $Parameter_def[[As]]) { $Primitive_deduced_defaultLibrary[[auto]] $LocalVariable_def[[VeryLongVariableName]] = 12312; $Class[[AS]] $LocalVariable_def[[AA]]; $Primitive_deduced_defaultLibrary[[auto]] $LocalVariable_def[[L]] = $LocalVariable[[AA]].$Field[[SomeMember]] $Operator[[+]] $Parameter[[A]]; auto $LocalVariable_def[[FN]] = [ $LocalVariable[[AA]]](int $Parameter_def[[A]]) -> void {}; $LocalVariable[[FN]]$Operator_userDefined[[(]]12312$Operator_userDefined[[)]]; } )cpp", R"cpp( void $Function_decl[[foo]](int); void $Function_decl[[Gah]](); void $Function_def[[foo]]() { auto $LocalVariable_def[[Bou]] = $Function[[Gah]]; } struct $Class_def[[A]] { void $Method_decl[[abc]](); }; )cpp", R"cpp( namespace $Namespace_decl[[abc]] { template struct $Class_def[[A]] { $TemplateParameter[[T]] $Field_decl[[t]]; }; } template struct $Class_def[[C]] : $Namespace[[abc]]::$Class[[A]]<$TemplateParameter[[T]]> { typename $TemplateParameter[[T]]::$Type_dependentName[[A]]* $Field_decl[[D]]; }; $Namespace[[abc]]::$Class[[A]] $Variable_def[[AA]]; typedef $Namespace[[abc]]::$Class[[A]] $Class_decl[[AAA]]; struct $Class_def[[B]] { $Class_decl_constrDestr[[B]](); ~$Class_decl_constrDestr[[B]](); void operator$Operator_decl[[<<]]($Class[[B]]); $Class[[AAA]] $Field_decl[[AA]]; }; $Class[[B]]::$Class_def_constrDestr[[B]]() {} $Class[[B]]::~$Class_def_constrDestr[[B]]() {} void $Function_def[[f]] () { $Class[[B]] $LocalVariable_def[[BB]] = $Class[[B]](); $LocalVariable[[BB]].~$Class_constrDestr[[B]](); $Class[[B]](); } )cpp", R"cpp( enum class $Enum_decl[[E]] { $EnumConstant_decl_readonly[[A]], $EnumConstant_decl_readonly[[B]], }; enum $Enum_decl[[EE]] { $EnumConstant_decl_readonly[[Hi]], }; struct $Class_def[[A]] { $Enum[[E]] $Field_decl[[EEE]]; $Enum[[EE]] $Field_decl[[EEEE]]; }; int $Variable_def[[I]] = $EnumConstant_readonly[[Hi]]; $Enum[[E]] $Variable_def[[L]] = $Enum[[E]]::$EnumConstant_readonly[[B]]; )cpp", R"cpp( namespace $Namespace_decl[[abc]] { namespace {} namespace $Namespace_decl[[bcd]] { struct $Class_def[[A]] {}; namespace $Namespace_decl[[cde]] { struct $Class_def[[A]] { enum class $Enum_decl[[B]] { $EnumConstant_decl_readonly[[Hi]], }; }; } } } using namespace $Namespace[[abc]]::$Namespace[[bcd]]; namespace $Namespace_decl[[vwz]] = $Namespace[[abc]]::$Namespace[[bcd]]::$Namespace[[cde]]; $Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable_def[[AA]]; $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]] $Variable_def[[AAA]] = $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]]::$EnumConstant_readonly[[Hi]]; ::$Namespace[[vwz]]::$Class[[A]] $Variable_def[[B]]; ::$Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable_def[[BB]]; )cpp", R"cpp( struct $Class_def[[D]] { double $Field_decl[[C]]; }; struct $Class_def[[A]] { double $Field_decl[[B]]; $Class[[D]] $Field_decl[[E]]; static double $StaticField_decl_static[[S]]; static void $StaticMethod_def_static[[bar]]() {} void $Method_def[[foo]]() { $Field[[B]] $Operator[[=]] 123; this->$Field[[B]] $Operator[[=]] 156; this->$Method[[foo]](); $Method[[foo]](); $StaticMethod_static[[bar]](); $StaticField_static[[S]] $Operator[[=]] 90.1; } }; void $Function_def[[foo]]() { $Class[[A]] $LocalVariable_def[[AA]]; $LocalVariable[[AA]].$Field[[B]] $Operator[[+=]] 2; $LocalVariable[[AA]].$Method[[foo]](); $LocalVariable[[AA]].$Field[[E]].$Field[[C]]; $Class[[A]]::$StaticField_static[[S]] $Operator[[=]] 90; } )cpp", R"cpp( struct $Class_def[[AA]] { int $Field_decl[[A]]; }; int $Variable_def[[B]]; $Class[[AA]] $Variable_def[[A]]{$Variable[[B]]}; )cpp", R"cpp( namespace $Namespace_decl[[a]] { struct $Class_def[[A]] {}; typedef char $Primitive_decl[[C]]; } typedef $Namespace[[a]]::$Class[[A]] $Class_decl[[B]]; using $Class_decl[[BB]] = $Namespace[[a]]::$Class[[A]]; enum class $Enum_decl[[E]] {}; typedef $Enum[[E]] $Enum_decl[[C]]; typedef $Enum[[C]] $Enum_decl[[CC]]; using $Enum_decl[[CD]] = $Enum[[CC]]; $Enum[[CC]] $Function_decl[[f]]($Class[[B]]); $Enum[[CD]] $Function_decl[[f]]($Class[[BB]]); typedef $Namespace[[a]]::$Primitive[[C]] $Primitive_decl[[PC]]; typedef float $Primitive_decl[[F]]; )cpp", R"cpp( template class $Class_def[[A]] { $TemplateParameter[[T]] $Field_decl[[AA]]; $TemplateParameter[[T]] $Method_decl[[foo]](); }; template class $Class_def[[B]] { $Class[[A]]<$TemplateParameter[[TT]]> $Field_decl[[AA]]; }; template class $Class_def[[BB]] {}; template class $Class_def[[BB]]<$TemplateParameter[[T]], int> {}; template class $Class_def[[BB]]<$TemplateParameter[[T]], $TemplateParameter[[T]]*> {}; template class $TemplateParameter_def[[T]], class $TemplateParameter_def[[C]]> $TemplateParameter[[T]]<$TemplateParameter[[C]]> $Function_decl[[f]](); template class $Class_def[[Foo]] {}; template void $Function_decl[[foo]]($TemplateParameter[[T]] ...); )cpp", R"cpp( template struct $Class_def[[Tmpl]] {$TemplateParameter[[T]] $Field_decl[[x]] = 0;}; extern template struct $Class_def[[Tmpl]]; template struct $Class_def[[Tmpl]]; )cpp", // This test is to guard against highlightings disappearing when using // conversion operators as their behaviour in the clang AST differ from // other CXXMethodDecls. R"cpp( class $Class_def[[Foo]] {}; struct $Class_def[[Bar]] { explicit operator $Class[[Foo]]*() const; explicit operator int() const; operator $Class[[Foo]](); }; void $Function_def[[f]]() { $Class[[Bar]] $LocalVariable_def[[B]]; $Class[[Foo]] $LocalVariable_def[[F]] = $LocalVariable[[B]]; $Class[[Foo]] *$LocalVariable_def[[FP]] = ($Class[[Foo]]*)$LocalVariable[[B]]; int $LocalVariable_def[[I]] = (int)$LocalVariable[[B]]; } )cpp", R"cpp( struct $Class_def[[B]] {}; struct $Class_def[[A]] { $Class[[B]] $Field_decl[[BB]]; $Class[[A]] &operator$Operator_decl[[=]]($Class[[A]] &&$Parameter_def[[O]]); }; $Class[[A]] &$Class[[A]]::operator$Operator_def[[=]]($Class[[A]] &&$Parameter_def[[O]]) = default; )cpp", R"cpp( enum $Enum_decl[[En]] { $EnumConstant_decl_readonly[[EC]], }; class $Class_def[[Foo]] {}; class $Class_def[[Bar]] { public: $Class[[Foo]] $Field_decl[[Fo]]; $Enum[[En]] $Field_decl[[E]]; int $Field_decl[[I]]; $Class_def_constrDestr[[Bar]] ($Class[[Foo]] $Parameter_def[[F]], $Enum[[En]] $Parameter_def[[E]]) : $Field[[Fo]] ($Parameter[[F]]), $Field[[E]] ($Parameter[[E]]), $Field[[I]] (123) {} }; class $Class_def[[Bar2]] : public $Class[[Bar]] { $Class_def_constrDestr[[Bar2]]() : $Class[[Bar]]($Class[[Foo]](), $EnumConstant_readonly[[EC]]) {} }; )cpp", R"cpp( enum $Enum_decl[[E]] { $EnumConstant_decl_readonly[[E]], }; class $Class_def[[Foo]] {}; $Enum_deduced[[auto]] $Variable_def[[AE]] = $Enum[[E]]::$EnumConstant_readonly[[E]]; $Class_deduced[[auto]] $Variable_def[[AF]] = $Class[[Foo]](); $Class_deduced[[decltype]](auto) $Variable_def[[AF2]] = $Class[[Foo]](); $Class_deduced[[auto]] *$Variable_def[[AFP]] = $Operator[[&]]$Variable[[AF]]; $Enum_deduced[[auto]] &$Variable_def[[AER]] = $Variable[[AE]]; $Primitive_deduced_defaultLibrary[[auto]] $Variable_def[[Form]] = 10.2 $Operator[[+]] 2 $Operator[[*]] 4; $Primitive_deduced_defaultLibrary[[decltype]]($Variable[[Form]]) $Variable_def[[F]] = 10; auto $Variable_def[[Fun]] = []()->void{}; )cpp", R"cpp( class $Class_def[[G]] {}; template<$Class[[G]] *$TemplateParameter_def_readonly[[U]]> class $Class_def[[GP]] {}; template<$Class[[G]] &$TemplateParameter_def_readonly[[U]]> class $Class_def[[GR]] {}; template class $Class_def[[IP]] { void $Method_def[[f]]() { $Operator[[*]]$TemplateParameter_readonly[[U]] $Operator[[+=]] 5; } }; template class $Class_def[[Foo]] { void $Method_def[[f]]() { for(int $LocalVariable_def[[I]] = 0; $LocalVariable[[I]] $Operator[[<]] $TemplateParameter_readonly[[U]];) {} } }; $Class[[G]] $Variable_def[[L]]; void $Function_def[[f]]() { $Class[[Foo]]<123> $LocalVariable_def[[F]]; $Class[[GP]]<$Operator[[&]]$Variable[[L]]> $LocalVariable_def[[LL]]; $Class[[GR]]<$Variable[[L]]> $LocalVariable_def[[LLL]]; } )cpp", R"cpp( template struct $Class_def[[G]] { void $Method_def[[foo]]( $TemplateParameter[[T]] *$Parameter_def[[O]]) { ($Parameter[[O]]$Operator_userDefined[[->*]]$TemplateParameter_readonly[[method]])(10); } }; struct $Class_def[[F]] { void $Method_decl[[f]](int); }; template struct $Class_def[[A]] { void $Method_def[[f]]() { ($Operator[[*]]$TemplateParameter_readonly[[Func]])(); } }; void $Function_def[[foo]]() { $Class[[F]] $LocalVariable_def[[FF]]; $Class[[G]]<$Class[[F]], $Operator[[&]]$Class[[F]]::$Method[[f]]> $LocalVariable_def[[GG]]; $LocalVariable[[GG]].$Method[[foo]]($Operator[[&]]$LocalVariable_usedAsMutablePointer[[FF]]); $Class[[A]]<$Function[[foo]]> $LocalVariable_def[[AA]]; } )cpp", // Tokens that share a source range but have conflicting Kinds are not // highlighted. R"cpp( #define $Macro_decl[[DEF_MULTIPLE]](X) namespace X { class X { int X; }; } #define $Macro_decl[[DEF_CLASS]](T) class T {}; // Preamble ends. $Macro[[DEF_MULTIPLE]](XYZ); $Macro[[DEF_MULTIPLE]](XYZW); $Macro[[DEF_CLASS]]($Class_def[[A]]) #define $Macro_decl[[MACRO_CONCAT]](X, V, T) T foo##X = V #define $Macro_decl[[DEF_VAR]](X, V) int X = V #define $Macro_decl[[DEF_VAR_T]](T, X, V) T X = V #define $Macro_decl[[DEF_VAR_REV]](V, X) DEF_VAR(X, V) #define $Macro_decl[[CPY]](X) X #define $Macro_decl[[DEF_VAR_TYPE]](X, Y) X Y #define $Macro_decl[[SOME_NAME]] variable #define $Macro_decl[[SOME_NAME_SET]] variable2 = 123 #define $Macro_decl[[INC_VAR]](X) X += 2 void $Function_def[[foo]]() { $Macro[[DEF_VAR]]($LocalVariable_def[[X]], 123); $Macro[[DEF_VAR_REV]](908, $LocalVariable_def[[XY]]); int $Macro[[CPY]]( $LocalVariable_def[[XX]] ); $Macro[[DEF_VAR_TYPE]]($Class[[A]], $LocalVariable_def[[AA]]); double $Macro[[SOME_NAME]]; int $Macro[[SOME_NAME_SET]]; $LocalVariable[[variable]] $Operator[[=]] 20.1; $Macro[[MACRO_CONCAT]](var, 2, float); $Macro[[DEF_VAR_T]]($Class[[A]], $Macro[[CPY]]( $Macro[[CPY]]($LocalVariable_def[[Nested]])), $Macro[[CPY]]($Class[[A]]())); $Macro[[INC_VAR]]($LocalVariable[[variable]]); } void $Macro[[SOME_NAME]](); $Macro[[DEF_VAR]]($Variable_def[[MMMMM]], 567); $Macro[[DEF_VAR_REV]](756, $Variable_def[[AB]]); #define $Macro_decl[[CALL_FN]](F) F(); #define $Macro_decl[[DEF_FN]](F) void F () $Macro[[DEF_FN]]($Function_def[[g]]) { $Macro[[CALL_FN]]($Function[[foo]]); } )cpp", R"cpp( #define $Macro_decl[[fail]](expr) expr #define $Macro_decl[[assert]](COND) if (!(COND)) { fail("assertion failed" #COND); } // Preamble ends. int $Variable_def[[x]]; int $Variable_def[[y]]; int $Function_decl[[f]](); void $Function_def[[foo]]() { $Macro[[assert]]($Variable[[x]] $Operator[[!=]] $Variable[[y]]); $Macro[[assert]]($Variable[[x]] $Operator[[!=]] $Function[[f]]()); } )cpp", // highlighting all macro references R"cpp( #ifndef $Macro[[name]] #define $Macro_decl[[name]] #endif #define $Macro_decl[[test]] #undef $Macro[[test]] $InactiveCode[[#ifdef test]] $InactiveCode[[#endif]] $InactiveCode[[#if defined(test)]] $InactiveCode[[#endif]] )cpp", R"cpp( struct $Class_def[[S]] { float $Field_decl[[Value]]; $Class[[S]] *$Field_decl[[Next]]; }; $Class[[S]] $Variable_def[[Global]][2] = {$Class[[S]](), $Class[[S]]()}; auto [$Variable_decl[[G1]], $Variable_decl[[G2]]] = $Variable[[Global]]; void $Function_def[[f]]($Class[[S]] $Parameter_def[[P]]) { int $LocalVariable_def[[A]][2] = {1,2}; auto [$LocalVariable_decl[[B1]], $LocalVariable_decl[[B2]]] = $LocalVariable[[A]]; auto [$LocalVariable_decl[[G1]], $LocalVariable_decl[[G2]]] = $Variable[[Global]]; $Class_deduced[[auto]] [$LocalVariable_decl[[P1]], $LocalVariable_decl[[P2]]] = $Parameter[[P]]; // Highlights references to BindingDecls. $LocalVariable[[B1]]$Operator[[++]]; } )cpp", R"cpp( template class $Class_def[[A]] { using $TemplateParameter_decl[[TemplateParam1]] = $TemplateParameter[[T]]; typedef $TemplateParameter[[T]] $TemplateParameter_decl[[TemplateParam2]]; using $Primitive_decl[[IntType]] = int; using $Typedef_decl[[Pointer]] = $TemplateParameter[[T]] *; using $Typedef_decl[[LVReference]] = $TemplateParameter[[T]] &; using $Typedef_decl[[RVReference]] = $TemplateParameter[[T]]&&; using $Typedef_decl[[Array]] = $TemplateParameter[[T]]*[3]; using $Typedef_decl[[MemberPointer]] = int ($Class[[A]]::*)(int); // Use various previously defined typedefs in a function type. void $Method_decl[[func]]( $Typedef[[Pointer]], $Typedef[[LVReference]], $Typedef[[RVReference]], $Typedef[[Array]], $Typedef[[MemberPointer]]); }; )cpp", R"cpp( template void $Function_decl[[phase1]]($TemplateParameter[[T]]); template void $Function_def[[foo]]($TemplateParameter[[T]] $Parameter_def[[P]]) { $Function[[phase1]]($Parameter[[P]]); $Unknown_dependentName[[phase2]]($Parameter[[P]]); } )cpp", R"cpp( class $Class_def[[A]] { template void $Method_decl[[bar]]($TemplateParameter[[T]]); }; template void $Function_def[[foo]]($TemplateParameter[[U]] $Parameter_def[[P]]) { $Class[[A]]().$Method[[bar]]($Parameter[[P]]); } )cpp", R"cpp( struct $Class_def[[A]] { template static void $StaticMethod_decl_static[[foo]]($TemplateParameter[[T]]); }; template struct $Class_def[[B]] { void $Method_def[[bar]]() { $Class[[A]]::$StaticMethod_static[[foo]]($TemplateParameter[[T]]()); } }; )cpp", R"cpp( template void $Function_decl[[foo]](typename $TemplateParameter[[T]]::$Type_dependentName[[Type]] = $TemplateParameter[[T]]::$Unknown_dependentName[[val]]); )cpp", R"cpp( template void $Function_def[[foo]]($TemplateParameter[[T]] $Parameter_def[[P]]) { $Parameter[[P]].$Unknown_dependentName[[Field]]; } )cpp", R"cpp( template class $Class_def[[A]] { int $Method_def[[foo]]() { return $TemplateParameter[[T]]::$Unknown_dependentName[[Field]]; } }; )cpp", // Highlighting the using decl as the underlying using shadow decl. R"cpp( void $Function_decl[[foo]](); using ::$Function[[foo]]; )cpp", // Highlighting of template template arguments. R"cpp( template