Debian Codegraph @

scintilla/lexilla/lexers/LexZig.cxx

raw
1 // Scintilla source code edit control
2 /** @file LexZig.cxx
3 ** Lexer for Zig language.
4 **/
5 // Based on Zufu Liu's Notepad4 Zig lexer
6 // Modified for Scintilla by Jiri Techet, 2024
7 // The License.txt file describes the conditions under which this software may be distributed.
8
9 #include <cassert>
10 #include <cstring>
11
12 #include <string>
13 #include <string_view>
14 #include <vector>
15 #include <map>
16 #include <algorithm>
17
18 #include "ILexer.h"
19 #include "Scintilla.h"
20 #include "SciLexer.h"
21
22 #include "WordList.h"
23 #include "LexAccessor.h"
24 #include "Accessor.h"
25 #include "StyleContext.h"
26 #include "CharacterSet.h"
27 #include "LexerModule.h"
28 #include "OptionSet.h"
29 #include "DefaultLexer.h"
30
31 using namespace Scintilla;
32 using namespace Lexilla;
33
34 namespace {
35 // Use an unnamed namespace to protect the functions and classes from name conflicts
36
37 constexpr bool IsAGraphic(int ch) noexcept {
38 // excludes C0 control characters and whitespace
39 return ch > 32 && ch < 127;
40 }
41
42 constexpr bool IsIdentifierStart(int ch) noexcept {
43 return IsUpperOrLowerCase(ch) || ch == '_';
44 }
45
46 constexpr bool IsIdentifierStartEx(int ch) noexcept {
47 return IsIdentifierStart(ch) || ch >= 0x80;
48 }
49
50 constexpr bool IsNumberStart(int ch, int chNext) noexcept {
51 return IsADigit(ch) || (ch == '.' && IsADigit(chNext));
52 }
53
54 constexpr bool IsIdentifierChar(int ch) noexcept {
55 return IsAlphaNumeric(ch) || ch == '_';
56 }
57
58 constexpr bool IsNumberContinue(int chPrev, int ch, int chNext) noexcept {
59 return ((ch == '+' || ch == '-') && (chPrev == 'e' || chPrev == 'E'))
60 || (ch == '.' && chNext != '.');
61 }
62
63 constexpr bool IsDecimalNumber(int chPrev, int ch, int chNext) noexcept {
64 return IsIdentifierChar(ch) || IsNumberContinue(chPrev, ch, chNext);
65 }
66
67 constexpr bool IsIdentifierCharEx(int ch) noexcept {
68 return IsIdentifierChar(ch) || ch >= 0x80;
69 }
70
71 // https://ziglang.org/documentation/master/#Escape-Sequences
72 struct EscapeSequence {
73 int outerState = SCE_ZIG_DEFAULT;
74 int digitsLeft = 0;
75 bool brace = false;
76
77 // highlight any character as escape sequence.
78 void resetEscapeState(int state, int chNext) noexcept {
79 outerState = state;
80 digitsLeft = 1;
81 brace = false;
82 if (chNext == 'x') {
83 digitsLeft = 3;
84 } else if (chNext == 'u') {
85 digitsLeft = 5;
86 }
87 }
88 void resetEscapeState(int state) noexcept {
89 outerState = state;
90 digitsLeft = 1;
91 brace = false;
92 }
93 bool atEscapeEnd(int ch) noexcept {
94 --digitsLeft;
95 return digitsLeft <= 0 || !IsAHeXDigit(ch);
96 }
97 };
98
99 enum {
100 ZigLineStateMaskLineComment = 1, // line comment
101 ZigLineStateMaskMultilineString = 1 << 1, // multiline string
102 };
103
104 struct FoldLineState {
105 int lineComment;
106 int multilineString;
107 constexpr explicit FoldLineState(int lineState) noexcept:
108 lineComment(lineState & ZigLineStateMaskLineComment),
109 multilineString((lineState >> 1) & 1) {
110 }
111 };
112
113 enum class KeywordType {
114 None = SCE_ZIG_DEFAULT,
115 Function = SCE_ZIG_FUNCTION,
116 };
117
118 enum {
119 KeywordIndex_Primary = 0,
120 KeywordIndex_Secondary = 1,
121 KeywordIndex_Tertiary = 2,
122 KeywordIndex_Type = 3,
123 };
124
125 // Options used for LexerZig
126 struct OptionsZig {
127 bool fold = false;
128 };
129
130 const char *const zigWordListDesc[] = {
131 "Primary keywords",
132 "Secondary keywords",
133 "Tertiary keywords",
134 "Global type definitions",
135 nullptr
136 };
137
138 struct OptionSetZig : public OptionSet<OptionsZig> {
139 OptionSetZig() {
140 DefineProperty("fold", &OptionsZig::fold);
141
142 DefineWordListSets(zigWordListDesc);
143 }
144 };
145
146 LexicalClass lexicalClasses[] = {
147 // Lexer ZIG SCLEX_ZIG SCE_ZIG_:
148 0, "SCE_ZIG_DEFAULT", "default", "White space",
149 1, "SCE_ZIG_COMMENTLINE", "comment line", "Comment: //",
150 2, "SCE_ZIG_COMMENTLINEDOC", "comment line documentation", "Comment: ///",
151 3, "SCE_ZIG_COMMENTLINETOP", "comment line documentation", "Comment: //!",
152 4, "SCE_ZIG_NUMBER", "literal numeric", "Number",
153 5, "SCE_ZIG_OPERATOR", "operator", "Operator",
154 6, "SCE_ZIG_CHARACTER", "literal string character", "Single quoted string",
155 7, "SCE_ZIG_STRING", "literal string", "Double quoted string",
156 8, "SCE_ZIG_MULTISTRING", "literal string multiline", "Multiline string introduced by two backslashes",
157 9, "SCE_ZIG_ESCAPECHAR", "literal string escapesequence", "Escape sequence",
158 10, "SCE_ZIG_IDENTIFIER", "identifier", "Identifier",
159 11, "SCE_ZIG_FUNCTION", "identifier", "Function definition",
160 12, "SCE_ZIG_BUILTIN_FUNCTION", "identifier", "Builtin function",
161 13, "SCE_ZIG_KW_PRIMARY", "keyword", "Primary keywords",
162 14, "SCE_ZIG_KW_SECONDARY", "identifier", "Secondary keywords",
163 15, "SCE_ZIG_KW_TERTIARY", "identifier", "Tertiary keywords",
164 16, "SCE_ZIG_KW_TYPE", "identifier", "Global types",
165 17, "SCE_ZIG_IDENTIFIER_STRING", "identifier", "Identifier using @\"\" syntax",
166 18, "SCE_ZIG_STRINGEOL", "error literal string", "End of line where string is not closed",
167 };
168
169 class LexerZig : public DefaultLexer {
170 WordList keywordsPrimary;
171 WordList keywordsSecondary;
172 WordList keywordsTertiary;
173 WordList keywordsTypes;
174 OptionsZig options;
175 OptionSetZig osZig;
176 public:
177 LexerZig(const char *languageName_, int language_) :
178 DefaultLexer(languageName_, language_, lexicalClasses, std::size(lexicalClasses)) {
179 }
180 // Deleted so LexerZig objects can not be copied.
181 LexerZig(const LexerZig &) = delete;
182 LexerZig(LexerZig &&) = delete;
183 void operator=(const LexerZig &) = delete;
184 void operator=(LexerZig &&) = delete;
185 ~LexerZig() override = default;
186
187 void SCI_METHOD Release() override {
188 delete this;
189 }
190 int SCI_METHOD Version() const override {
191 return lvRelease5;
192 }
193 const char *SCI_METHOD PropertyNames() override {
194 return osZig.PropertyNames();
195 }
196 int SCI_METHOD PropertyType(const char *name) override {
197 return osZig.PropertyType(name);
198 }
199 const char *SCI_METHOD DescribeProperty(const char *name) override {
200 return osZig.DescribeProperty(name);
201 }
202 Sci_Position SCI_METHOD PropertySet(const char *key, const char *val) override;
203 const char *SCI_METHOD PropertyGet(const char *key) override {
204 return osZig.PropertyGet(key);
205 }
206 const char *SCI_METHOD DescribeWordListSets() override {
207 return osZig.DescribeWordListSets();
208 }
209 Sci_Position SCI_METHOD WordListSet(int n, const char *wl) override;
210
211 void SCI_METHOD Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override;
212 void SCI_METHOD Fold(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override;
213
214 void *SCI_METHOD PrivateCall(int, void *) override {
215 return nullptr;
216 }
217
218 void BacktrackToStart(const LexAccessor &styler, int stateMask, Sci_PositionU &startPos, Sci_Position &lengthDoc, int &initStyle);
219 Sci_PositionU LookbackNonWhite(LexAccessor &styler, Sci_PositionU startPos, int &chPrevNonWhite, int &stylePrevNonWhite);
220
221 static ILexer5 *LexerFactoryZig() {
222 return new LexerZig("zig", SCLEX_ZIG);
223 }
224 };
225
226 Sci_Position SCI_METHOD LexerZig::PropertySet(const char *key, const char *val) {
227 if (osZig.PropertySet(&options, key, val)) {
228 return 0;
229 }
230 return -1;
231 }
232
233 Sci_Position SCI_METHOD LexerZig::WordListSet(int n, const char *wl) {
234 WordList *wordListN = nullptr;
235 switch (n) {
236 case 0:
237 wordListN = &keywordsPrimary;
238 break;
239 case 1:
240 wordListN = &keywordsSecondary;
241 break;
242 case 2:
243 wordListN = &keywordsTertiary;
244 break;
245 case 3:
246 wordListN = &keywordsTypes;
247 break;
248 default:
249 break;
250 }
251 Sci_Position firstModification = -1;
252 if (wordListN && wordListN->Set(wl, false)) {
253 firstModification = 0;
254 }
255 return firstModification;
256 }
257
258 void LexerZig::Lex(Sci_PositionU startPos, Sci_Position lengthDoc, int initStyle, IDocument *pAccess) {
259 Accessor styler(pAccess, nullptr);
260
261 KeywordType kwType = KeywordType::None;
262 int visibleChars = 0;
263 int lineState = 0;
264 EscapeSequence escSeq;
265
266 if (initStyle == SCE_ZIG_STRINGEOL) {
267 initStyle = SCE_ZIG_DEFAULT;
268 }
269
270 StyleContext sc(startPos, lengthDoc, initStyle, styler);
271
272 while (sc.More()) {
273 switch (sc.state) {
274 case SCE_ZIG_OPERATOR:
275 sc.SetState(SCE_ZIG_DEFAULT);
276 break;
277
278 case SCE_ZIG_NUMBER:
279 if (!IsDecimalNumber(sc.chPrev, sc.ch, sc.chNext)) {
280 sc.SetState(SCE_ZIG_DEFAULT);
281 }
282 break;
283
284 case SCE_ZIG_IDENTIFIER:
285 case SCE_ZIG_BUILTIN_FUNCTION:
286 if (!IsIdentifierCharEx(sc.ch)) {
287 if (sc.state == SCE_ZIG_IDENTIFIER) {
288 char s[64];
289 sc.GetCurrent(s, sizeof(s));
290 if (kwType != KeywordType::None) {
291 sc.ChangeState(static_cast<int>(kwType));
292 } else if (keywordsPrimary.InList(s)) {
293 sc.ChangeState(SCE_ZIG_KW_PRIMARY);
294 kwType = KeywordType::None;
295 if (strcmp(s, "fn") == 0) {
296 kwType = KeywordType::Function;
297 }
298 } else if (keywordsSecondary.InList(s)) {
299 sc.ChangeState(SCE_ZIG_KW_SECONDARY);
300 } else if (keywordsTertiary.InList(s)) {
301 sc.ChangeState(SCE_ZIG_KW_TERTIARY);
302 } else if (keywordsTypes.InList(s)) {
303 sc.ChangeState(SCE_ZIG_KW_TYPE);
304 }
305 }
306 if (sc.state != SCE_ZIG_KW_PRIMARY) {
307 kwType = KeywordType::None;
308 }
309 sc.SetState(SCE_ZIG_DEFAULT);
310 }
311 break;
312
313 case SCE_ZIG_CHARACTER:
314 case SCE_ZIG_STRING:
315 case SCE_ZIG_MULTISTRING:
316 case SCE_ZIG_IDENTIFIER_STRING:
317 if (sc.atLineStart) {
318 sc.SetState(SCE_ZIG_DEFAULT);
319 } else if (sc.atLineEnd && sc.state != SCE_ZIG_MULTISTRING) {
320 sc.ChangeState(SCE_ZIG_STRINGEOL);
321 } else if (sc.ch == '\\' && sc.state != SCE_ZIG_MULTISTRING) {
322 escSeq.resetEscapeState(sc.state, sc.chNext);
323 sc.SetState(SCE_ZIG_ESCAPECHAR);
324 sc.Forward();
325 if (sc.Match('u', '{')) {
326 escSeq.brace = true;
327 escSeq.digitsLeft = 9;
328 sc.Forward();
329 }
330 } else if ((sc.ch == '\'' && sc.state == SCE_ZIG_CHARACTER) ||
331 (sc.ch == '\"' && (sc.state == SCE_ZIG_STRING || sc.state == SCE_ZIG_IDENTIFIER_STRING))) {
332 sc.ForwardSetState(SCE_ZIG_DEFAULT);
333 }
334 break;
335
336 case SCE_ZIG_STRINGEOL:
337 if (sc.atLineStart) {
338 sc.SetState(SCE_ZIG_DEFAULT);
339 }
340 break;
341
342 case SCE_ZIG_ESCAPECHAR:
343 if (escSeq.atEscapeEnd(sc.ch)) {
344 if (escSeq.brace && sc.ch == '}') {
345 sc.Forward();
346 }
347 sc.SetState(escSeq.outerState);
348 continue;
349 }
350 break;
351
352 case SCE_ZIG_COMMENTLINE:
353 case SCE_ZIG_COMMENTLINEDOC:
354 case SCE_ZIG_COMMENTLINETOP:
355 if (sc.atLineStart) {
356 sc.SetState(SCE_ZIG_DEFAULT);
357 }
358 break;
359 }
360
361 if (sc.state == SCE_ZIG_DEFAULT) {
362 if (sc.Match('/', '/')) {
363 if (visibleChars == 0) {
364 lineState = ZigLineStateMaskLineComment;
365 }
366 sc.SetState(SCE_ZIG_COMMENTLINE);
367 sc.Forward(2);
368 if (sc.ch == '!') {
369 sc.ChangeState(SCE_ZIG_COMMENTLINETOP);
370 } else if (sc.ch == '/' && sc.chNext != '/') {
371 sc.ChangeState(SCE_ZIG_COMMENTLINEDOC);
372 }
373 } else if (sc.Match('\\', '\\')) {
374 lineState = ZigLineStateMaskMultilineString;
375 sc.SetState(SCE_ZIG_MULTISTRING);
376 } else if (sc.ch == '\"') {
377 sc.SetState(SCE_ZIG_STRING);
378 } else if (sc.ch == '\'') {
379 sc.SetState(SCE_ZIG_CHARACTER);
380 } else if (IsNumberStart(sc.ch, sc.chNext)) {
381 sc.SetState(SCE_ZIG_NUMBER);
382 } else if ((sc.ch == '@' && IsIdentifierStartEx(sc.chNext)) || IsIdentifierStartEx(sc.ch)) {
383 sc.SetState((sc.ch == '@') ? SCE_ZIG_BUILTIN_FUNCTION : SCE_ZIG_IDENTIFIER);
384 } else if (sc.ch == '@' && sc.chNext == '"') {
385 sc.SetState(SCE_ZIG_IDENTIFIER_STRING);
386 sc.Forward();
387 } else if (IsAGraphic(sc.ch)) {
388 sc.SetState(SCE_ZIG_OPERATOR);
389 }
390 }
391
392 if (visibleChars == 0 && !isspacechar(sc.ch)) {
393 visibleChars++;
394 }
395 if (sc.atLineEnd) {
396 styler.SetLineState(sc.currentLine, lineState);
397 lineState = 0;
398 kwType = KeywordType::None;
399 visibleChars = 0;
400 }
401 sc.Forward();
402 }
403
404 sc.Complete();
405 }
406
407 void LexerZig::Fold(Sci_PositionU startPos, Sci_Position lengthDoc, int initStyle, IDocument *pAccess) {
408 if (!options.fold)
409 return;
410
411 Accessor styler(pAccess, nullptr);
412 const Sci_PositionU endPos = startPos + lengthDoc;
413 Sci_Position lineCurrent = styler.GetLine(startPos);
414 while (lineCurrent > 0) {
415 lineCurrent--;
416 startPos = styler.LineStart(lineCurrent);
417 initStyle = (startPos > 0) ? styler.StyleIndexAt(startPos) : 0;
418 if (!AnyOf(initStyle, SCE_ZIG_MULTISTRING,
419 SCE_ZIG_COMMENTLINE, SCE_ZIG_COMMENTLINEDOC, SCE_ZIG_COMMENTLINETOP)) {
420 break;
421 }
422 }
423 FoldLineState foldPrev(0);
424 int levelCurrent = SC_FOLDLEVELBASE;
425 if (lineCurrent > 0) {
426 levelCurrent = styler.LevelAt(lineCurrent - 1) >> 16;
427 foldPrev = FoldLineState(styler.GetLineState(lineCurrent - 1));
428 }
429
430 int levelNext = levelCurrent;
431 FoldLineState foldCurrent(styler.GetLineState(lineCurrent));
432 Sci_PositionU lineStartNext = styler.LineStart(lineCurrent + 1);
433 lineStartNext = std::min(lineStartNext, endPos);
434
435 while (startPos < endPos) {
436 initStyle = styler.StyleIndexAt(startPos);
437
438 if (initStyle == SCE_ZIG_OPERATOR) {
439 const char ch = styler[startPos];
440 if (ch == '{' || ch == '[' || ch == '(') {
441 levelNext++;
442 } else if (ch == '}' || ch == ']' || ch == ')') {
443 levelNext--;
444 }
445 }
446
447 ++startPos;
448 if (startPos == lineStartNext) {
449 const FoldLineState foldNext(styler.GetLineState(lineCurrent + 1));
450 levelNext = std::max(levelNext, SC_FOLDLEVELBASE);
451 if (foldCurrent.lineComment) {
452 levelNext += foldNext.lineComment - foldPrev.lineComment;
453 } else if (foldCurrent.multilineString) {
454 levelNext += foldNext.multilineString - foldPrev.multilineString;
455 }
456
457 const int levelUse = levelCurrent;
458 int lev = levelUse | (levelNext << 16);
459 if (levelUse < levelNext) {
460 lev |= SC_FOLDLEVELHEADERFLAG;
461 }
462 styler.SetLevel(lineCurrent, lev);
463
464 lineCurrent++;
465 lineStartNext = styler.LineStart(lineCurrent + 1);
466 lineStartNext = std::min(lineStartNext, endPos);
467 levelCurrent = levelNext;
468 foldPrev = foldCurrent;
469 foldCurrent = foldNext;
470 }
471 }
472 }
473
474 } // unnamed namespace end
475
476 extern const LexerModule lmZig(SCLEX_ZIG, LexerZig::LexerFactoryZig, "zig", zigWordListDesc);