Debian Codegraph @

scintilla/lexilla/lexers/LexNix.cxx

raw
1 // Scintilla source code edit control
2 /** @file LexNix.cxx
3 ** Lexer for Nix.
4 **/
5 // Based on Zufu Liu's Notepad4 Dart lexer
6 // Modified for Nix and Scintilla by Jiri Techet, 2024
7 // The License.txt file describes the conditions under which this software may be distributed.
8
9 #include <cassert>
10 #include <cstring>
11
12 #include <string>
13 #include <string_view>
14 #include <vector>
15 #include <map>
16 #include <algorithm>
17
18 #include "ILexer.h"
19 #include "Scintilla.h"
20 #include "SciLexer.h"
21
22 #include "WordList.h"
23 #include "LexAccessor.h"
24 #include "Accessor.h"
25 #include "StyleContext.h"
26 #include "CharacterSet.h"
27 #include "LexerModule.h"
28 #include "OptionSet.h"
29 #include "DefaultLexer.h"
30
31 using namespace Scintilla;
32 using namespace Lexilla;
33
34 namespace {
35 // Use an unnamed namespace to protect the functions and classes from name conflicts
36
37 constexpr bool IsAGraphic(int ch) noexcept {
38 // excludes C0 control characters and whitespace
39 return ch > 32 && ch < 127;
40 }
41
42 constexpr bool IsIdentifierChar(int ch) noexcept {
43 return IsAlphaNumeric(ch) || ch == '_' || ch == '\'' || ch == '-';
44 }
45
46 constexpr bool IsIdentifierStart(int ch) noexcept {
47 return IsUpperOrLowerCase(ch) || ch == '_';
48 }
49
50 constexpr bool IsNumberContinue(int chPrev, int ch, int chNext) noexcept {
51 return ((ch == '+' || ch == '-') && (chPrev == 'e' || chPrev == 'E'))
52 || (ch == '.' && chNext != '.');
53 }
54
55 constexpr bool IsNumberStart(int ch, int chNext) noexcept {
56 return IsADigit(ch) || (ch == '.' && IsADigit(chNext));
57 }
58
59 constexpr bool IsDecimalNumber(int chPrev, int ch, int chNext) noexcept {
60 return IsIdentifierChar(ch) || IsNumberContinue(chPrev, ch, chNext);
61 }
62
63 constexpr bool IsNixIdentifierStart(int ch) noexcept {
64 return IsIdentifierStart(ch);
65 }
66
67 constexpr bool IsNixIdentifierChar(int ch) noexcept {
68 return IsIdentifierChar(ch);
69 }
70
71 constexpr bool IsPathTerminator(int ch) noexcept {
72 return isspacechar(ch) || AnyOf(ch, ';', ',', ']', '}', ')');
73 }
74
75 bool IsKey(LexAccessor &styler, Sci_Position start) {
76 for (Sci_Position i = 0; i < 50; i++) {
77 char curr = styler.SafeGetCharAt(start+i, '\0');
78 char next = styler.SafeGetCharAt(start+i+1, '\0');
79 bool atEOL = (curr == '\r' && next != '\n') || (curr == '\n') || (curr == '\0');
80 if (curr == '=') {
81 return true;
82 } else if (!isspacechar(curr) || atEOL) {
83 return false;
84 }
85 }
86 return false;
87 }
88
89 bool IsPath(LexAccessor &styler, Sci_Position start) {
90 for (Sci_Position i = 0; i < 50; i++) {
91 char curr = styler.SafeGetCharAt(start+i, '\0');
92 char next = styler.SafeGetCharAt(start+i+1, '\0');
93 bool atEOL = (curr == '\r' && next != '\n') || (curr == '\n') || (curr == '\0');
94 if (curr == '/') {
95 return true;
96 } else if (IsPathTerminator(curr) || curr == '$' || atEOL) {
97 return false;
98 }
99 }
100 return false;
101 }
102
103 enum {
104 NixLineStateMaskInterpolation = 1, // string interpolation
105 };
106
107 // string interpolating state
108 struct InterpolatingState {
109 int state;
110 int braceCount;
111 };
112
113 // Options used for LexerNix
114 struct OptionsNix {
115 bool fold = false;
116 };
117
118 const char * const nixWordListDesc[] = {
119 "Keywords 1",
120 "Keywords 2",
121 "Keywords 3",
122 "Keywords 4",
123 nullptr
124 };
125
126 struct OptionSetNix : public OptionSet<OptionsNix> {
127 OptionSetNix() {
128 DefineProperty("fold", &OptionsNix::fold);
129
130 DefineWordListSets(nixWordListDesc);
131 }
132 };
133
134
135 LexicalClass lexicalClasses[] = {
136 // Lexer NIX SCLEX_NIX SCE_NIX_:
137 0, "SCE_NIX_DEFAULT", "default", "White space",
138 1, "SCE_NIX_COMMENTLINE", "comment line", "Comment: //",
139 2, "SCE_NIX_COMMENTBLOCK", "comment", "Comment: /* */",
140 3, "SCE_NIX_STRING", "literal string", "Double quoted string",
141 4, "SCE_NIX_STRING_MULTILINE", "literal string multiline", "Single quoted multiline string",
142 5, "SCE_NIX_ESCAPECHAR", "literal string escapesequence", "Escape sequence",
143 6, "SCE_NIX_IDENTIFIER", "identifier", "Identifier",
144 7, "SCE_NIX_OPERATOR", "operator", "Operator",
145 8, "SCE_NIX_OPERATOR_STRING", "operator interpolated", "Braces following $ inside string",
146 9, "SCE_NIX_NUMBER", "literal numeric", "Number",
147 10, "SCE_NIX_KEY", "key", "Keys preceding '='",
148 11, "SCE_NIX_PATH", "identifier", "Path literal",
149 12, "SCE_NIX_KEYWORD1", "keyword", "Primary keywords",
150 13, "SCE_NIX_KEYWORD2", "identifier", "Keywords 2",
151 14, "SCE_NIX_KEYWORD3", "identifier", "Keywords 3",
152 15, "SCE_NIX_KEYWORD4", "identifier", "Keywords 4",
153 16, "SCE_NIX_STRINGEOL", "error literal string", "End of line where string is not closed",
154 };
155
156 class LexerNix : public DefaultLexer {
157 WordList keywordsPrimary;
158 WordList keywordsSecondary;
159 WordList keywordsTertiary;
160 WordList keywordsTypes;
161 OptionsNix options;
162 OptionSetNix osNix;
163 public:
164 LexerNix(const char *languageName_, int language_) :
165 DefaultLexer(languageName_, language_, lexicalClasses, std::size(lexicalClasses)) {
166 }
167 // Deleted so LexerNix objects can not be copied.
168 LexerNix(const LexerNix &) = delete;
169 LexerNix(LexerNix &&) = delete;
170 void operator=(const LexerNix &) = delete;
171 void operator=(LexerNix &&) = delete;
172 ~LexerNix() override = default;
173
174 void SCI_METHOD Release() override {
175 delete this;
176 }
177 int SCI_METHOD Version() const override {
178 return lvRelease5;
179 }
180 const char *SCI_METHOD PropertyNames() override {
181 return osNix.PropertyNames();
182 }
183 int SCI_METHOD PropertyType(const char *name) override {
184 return osNix.PropertyType(name);
185 }
186 const char *SCI_METHOD DescribeProperty(const char *name) override {
187 return osNix.DescribeProperty(name);
188 }
189 Sci_Position SCI_METHOD PropertySet(const char *key, const char *val) override;
190 const char *SCI_METHOD PropertyGet(const char *key) override {
191 return osNix.PropertyGet(key);
192 }
193 const char *SCI_METHOD DescribeWordListSets() override {
194 return osNix.DescribeWordListSets();
195 }
196 Sci_Position SCI_METHOD WordListSet(int n, const char *wl) override;
197
198 void SCI_METHOD Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override;
199 void SCI_METHOD Fold(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override;
200
201 void *SCI_METHOD PrivateCall(int, void *) override {
202 return nullptr;
203 }
204
205 void BacktrackToStart(const LexAccessor &styler, int stateMask, Sci_PositionU &startPos, Sci_Position &lengthDoc, int &initStyle);
206
207 static ILexer5 *LexerFactoryNix() {
208 return new LexerNix("nix", SCLEX_NIX);
209 }
210 };
211
212 Sci_Position SCI_METHOD LexerNix::PropertySet(const char *key, const char *val) {
213 if (osNix.PropertySet(&options, key, val)) {
214 return 0;
215 }
216 return -1;
217 }
218
219 Sci_Position SCI_METHOD LexerNix::WordListSet(int n, const char *wl) {
220 WordList *wordListN = nullptr;
221 switch (n) {
222 case 0:
223 wordListN = &keywordsPrimary;
224 break;
225 case 1:
226 wordListN = &keywordsSecondary;
227 break;
228 case 2:
229 wordListN = &keywordsTertiary;
230 break;
231 case 3:
232 wordListN = &keywordsTypes;
233 break;
234 default:
235 break;
236 }
237 Sci_Position firstModification = -1;
238 if (wordListN && wordListN->Set(wl/*, false*/)) {
239 firstModification = 0;
240 }
241 return firstModification;
242 }
243
244 void LexerNix::BacktrackToStart(const LexAccessor &styler, int stateMask, Sci_PositionU &startPos, Sci_Position &lengthDoc, int &initStyle) {
245 const Sci_Position currentLine = styler.GetLine(startPos);
246 if (currentLine != 0) {
247 Sci_Position line = currentLine - 1;
248 int lineState = styler.GetLineState(line);
249 while ((lineState & stateMask) != 0 && line != 0) {
250 --line;
251 lineState = styler.GetLineState(line);
252 }
253 if ((lineState & stateMask) == 0) {
254 ++line;
255 }
256 if (line != currentLine) {
257 const Sci_PositionU endPos = startPos + lengthDoc;
258 startPos = (line == 0) ? 0 : styler.LineStart(line);
259 lengthDoc = endPos - startPos;
260 initStyle = (startPos == 0) ? 0 : styler.StyleAt(startPos - 1);
261 }
262 }
263 }
264
265 void LexerNix::Lex(Sci_PositionU startPos, Sci_Position lengthDoc, int initStyle, IDocument *pAccess) {
266 Accessor styler(pAccess, nullptr);
267
268 std::vector<InterpolatingState> interpolatingStack;
269
270 if (initStyle == SCE_NIX_STRINGEOL) {
271 initStyle = SCE_NIX_DEFAULT;
272 }
273
274 if (startPos != 0) {
275 // backtrack to the line where interpolation starts
276 BacktrackToStart(styler, NixLineStateMaskInterpolation, startPos, lengthDoc, initStyle);
277 }
278
279 StyleContext sc(startPos, lengthDoc, initStyle, styler);
280
281 while (sc.More()) {
282 switch (sc.state) {
283 case SCE_NIX_OPERATOR:
284 case SCE_NIX_OPERATOR_STRING:
285 sc.SetState(SCE_NIX_DEFAULT);
286 break;
287
288 case SCE_NIX_NUMBER:
289 if (!IsDecimalNumber(sc.chPrev, sc.ch, sc.chNext)) {
290 sc.SetState(SCE_NIX_DEFAULT);
291 }
292 break;
293
294 case SCE_NIX_IDENTIFIER:
295 if (sc.state == SCE_NIX_IDENTIFIER && !IsNixIdentifierChar(sc.ch)) {
296 char s[64];
297 sc.GetCurrent(s, sizeof(s));
298 if (IsKey(styler, sc.currentPos)) {
299 sc.ChangeState(SCE_NIX_KEY);
300 } else if (keywordsPrimary.InList(s)) {
301 sc.ChangeState(SCE_NIX_KEYWORD1);
302 } else if (keywordsSecondary.InList(s)) {
303 sc.ChangeState(SCE_NIX_KEYWORD2);
304 } else if (keywordsTertiary.InList(s)) {
305 sc.ChangeState(SCE_NIX_KEYWORD3);
306 } else if (keywordsTypes.InList(s)) {
307 sc.ChangeState(SCE_NIX_KEYWORD4);
308 }
309
310 sc.SetState(SCE_NIX_DEFAULT);
311 }
312 break;
313
314 case SCE_NIX_COMMENTLINE:
315 if (sc.atLineStart) {
316 sc.SetState(SCE_NIX_DEFAULT);
317 }
318 break;
319
320 case SCE_NIX_COMMENTBLOCK:
321 if (sc.Match('*', '/')) {
322 sc.Forward(2);
323 sc.SetState(SCE_NIX_DEFAULT);
324 continue;
325 }
326 break;
327
328 case SCE_NIX_STRING:
329 if (sc.atLineStart) {
330 sc.SetState(SCE_NIX_DEFAULT);
331 } else if (sc.atLineEnd) {
332 sc.ChangeState(SCE_NIX_STRINGEOL);
333 } else if (sc.ch == '\\' && AnyOf(sc.chNext, '"', '\\', 'n', 'r', 't', '$')) {
334 sc.SetState(SCE_NIX_STRING);
335 sc.ChangeState(SCE_NIX_ESCAPECHAR);
336 sc.Forward(2);
337 sc.SetState(SCE_NIX_STRING);
338 continue;
339 } else if (sc.ch == '\"') {
340 sc.Forward();
341 sc.SetState(SCE_NIX_DEFAULT);
342 continue;
343 } else if (sc.Match('$','$')) {
344 sc.Forward();
345 continue;
346 }
347 break;
348
349 case SCE_NIX_STRINGEOL:
350 if (sc.atLineStart) {
351 sc.SetState(SCE_NIX_DEFAULT);
352 }
353 break;
354
355 case SCE_NIX_PATH:
356 if (sc.atLineStart) {
357 sc.SetState(SCE_NIX_DEFAULT);
358 } else if (sc.ch == '>') {
359 sc.Forward();
360 sc.SetState(SCE_NIX_DEFAULT);
361 continue;
362 } else if (IsPathTerminator(sc.ch)) {
363 sc.SetState(SCE_NIX_DEFAULT);
364 }
365 break;
366
367 case SCE_NIX_STRING_MULTILINE:
368 if (sc.ch == '\'' && sc.chNext == '\'') {
369 if (AnyOf(styler.SafeGetCharAt(sc.currentPos+2, '\0'), '$', '\'', '\\')) {
370 sc.SetState(SCE_NIX_ESCAPECHAR);
371 sc.Forward(2);
372 if (sc.ch == '$' || sc.ch == '\'') {
373 sc.Forward();
374 } else if (sc.ch == '\\') {
375 sc.Forward(2);
376 }
377 sc.SetState(SCE_NIX_STRING_MULTILINE);
378 continue;
379 } else {
380 sc.Forward(2);
381 sc.SetState(SCE_NIX_DEFAULT);
382 continue;
383 }
384 } else if (sc.Match('$','$')) {
385 sc.Forward();
386 continue;
387 }
388 break;
389 }
390
391 if (sc.state == SCE_NIX_DEFAULT)
392 {
393 if (sc.ch == '/' && sc.chNext == '*') {
394 sc.SetState(SCE_NIX_COMMENTBLOCK);
395 sc.Forward();
396 continue;
397 } else if (sc.ch == '#') {
398 sc.SetState(SCE_NIX_COMMENTLINE);
399 } else if (sc.ch == '"') {
400 sc.SetState(SCE_NIX_STRING);
401 } else if (sc.ch == '\'' && sc.chNext == '\'') {
402 sc.SetState(SCE_NIX_STRING_MULTILINE);
403 sc.Forward();
404 continue;
405 } else if (IsNumberStart(sc.ch, sc.chNext)) {
406 sc.SetState(SCE_NIX_NUMBER);
407 } else if (sc.ch == '<') {
408 sc.SetState(SCE_NIX_PATH);
409 } else if ((IsNixIdentifierStart(sc.ch) || AnyOf(sc.ch, '~', '.', '/')) &&
410 IsPath(styler, sc.currentPos)) {
411 sc.SetState(SCE_NIX_PATH);
412 } else if (IsNixIdentifierStart(sc.ch)) {
413 sc.SetState(SCE_NIX_IDENTIFIER);
414 } else if (IsAGraphic(sc.ch) && sc.ch != '$') {
415 sc.SetState(SCE_NIX_OPERATOR);
416 if (!interpolatingStack.empty() && AnyOf(sc.ch, '{', '}')) {
417 InterpolatingState &current = interpolatingStack.back();
418 if (sc.ch == '{') {
419 current.braceCount += 1;
420 } else {
421 current.braceCount -= 1;
422 if (current.braceCount == 0) {
423 sc.ChangeState(SCE_NIX_OPERATOR_STRING);
424 sc.ForwardSetState(current.state);
425 interpolatingStack.pop_back();
426 continue;
427 }
428 }
429 }
430 }
431 }
432
433 // string interpolations
434 if (AnyOf(sc.state, SCE_NIX_DEFAULT, SCE_NIX_STRING, SCE_NIX_STRING_MULTILINE, SCE_NIX_PATH)) {
435 if (sc.Match('$','{')) {
436 interpolatingStack.push_back({sc.state, 1});
437 sc.SetState(SCE_NIX_OPERATOR_STRING);
438 sc.Forward();
439 }
440 }
441
442 if (sc.atLineEnd) {
443 int lineState = 0;
444 if (!interpolatingStack.empty()) {
445 lineState |= NixLineStateMaskInterpolation;
446 }
447 styler.SetLineState(sc.currentLine, lineState);
448 }
449
450 sc.Forward();
451 }
452
453 sc.Complete();
454 }
455
456 void LexerNix::Fold(Sci_PositionU startPos, Sci_Position lengthDoc, int initStyle, IDocument *pAccess) {
457 if (!options.fold)
458 return;
459
460 Accessor styler(pAccess, nullptr);
461 const Sci_PositionU endPos = startPos + lengthDoc;
462 Sci_Position lineCurrent = styler.GetLine(startPos);
463 while (lineCurrent > 0) {
464 lineCurrent--;
465 startPos = styler.LineStart(lineCurrent);
466 initStyle = (startPos > 0) ? styler.StyleIndexAt(startPos) : 0;
467 if (!AnyOf(initStyle, SCE_NIX_COMMENTBLOCK, SCE_NIX_STRING_MULTILINE)) {
468 break;
469 }
470 }
471 int levelCurrent = SC_FOLDLEVELBASE;
472 if (lineCurrent > 0) {
473 levelCurrent = styler.LevelAt(lineCurrent - 1) >> 16;
474 }
475
476 int levelNext = levelCurrent;
477 Sci_PositionU lineStartNext = styler.LineStart(lineCurrent + 1);
478 lineStartNext = std::min(lineStartNext, endPos);
479
480 char chNext = styler[startPos];
481 int styleNext = styler.StyleIndexAt(startPos);
482 int style = initStyle;
483
484 while (startPos < endPos) {
485 const char ch = chNext;
486 const int stylePrev = style;
487 style = styleNext;
488 chNext = styler[++startPos];
489 styleNext = styler.StyleIndexAt(startPos);
490
491 switch (style) {
492 case SCE_NIX_COMMENTBLOCK:
493 if (style != stylePrev) {
494 levelNext++;
495 }
496 if (style != styleNext) {
497 levelNext--;
498 }
499 break;
500
501 case SCE_NIX_STRING_MULTILINE:
502 if (style != stylePrev && !AnyOf(stylePrev, SCE_NIX_ESCAPECHAR, SCE_NIX_OPERATOR_STRING)) {
503 levelNext++;
504 }
505 if (style != styleNext && !AnyOf(styleNext, SCE_NIX_ESCAPECHAR, SCE_NIX_OPERATOR_STRING)) {
506 levelNext--;
507 }
508 break;
509
510 case SCE_NIX_OPERATOR:
511 case SCE_NIX_OPERATOR_STRING:
512 if (ch == '{' || ch == '[' || ch == '(') {
513 levelNext++;
514 } else if (ch == '}' || ch == ']' || ch == ')') {
515 levelNext--;
516 }
517 break;
518 }
519
520 if (startPos == lineStartNext) {
521 levelNext = std::max(levelNext, SC_FOLDLEVELBASE);
522
523 const int levelUse = levelCurrent;
524 int lev = levelUse | (levelNext << 16);
525 if (levelUse < levelNext) {
526 lev |= SC_FOLDLEVELHEADERFLAG;
527 }
528 styler.SetLevel(lineCurrent, lev);
529
530 lineCurrent++;
531 lineStartNext = styler.LineStart(lineCurrent + 1);
532 lineStartNext = std::min(lineStartNext, endPos);
533 levelCurrent = levelNext;
534 }
535 }
536 }
537
538 } // unnamed namespace end
539
540 extern const LexerModule lmNix(SCLEX_NIX, LexerNix::LexerFactoryNix, "nix", nixWordListDesc);