Debian Codegraph @

scintilla/lexilla/lexers/LexTOML.cxx

raw
1 // Scintilla source code edit control
2 /** @file LexTOML.cxx
3 ** Lexer for TOML language.
4 **/
5 // Based on Zufu Liu's Notepad4 TOML lexer
6 // Modified for Scintilla by Jiri Techet, 2024
7 // The License.txt file describes the conditions under which this software may be distributed.
8
9 #include <cassert>
10 #include <cstring>
11
12 #include <string>
13 #include <string_view>
14
15 #include "ILexer.h"
16 #include "Scintilla.h"
17 #include "SciLexer.h"
18
19 #include "WordList.h"
20 #include "LexAccessor.h"
21 #include "Accessor.h"
22 #include "StyleContext.h"
23 #include "CharacterSet.h"
24 #include "LexerModule.h"
25
26 using namespace Lexilla;
27
28 namespace {
29 // Use an unnamed namespace to protect the functions and classes from name conflicts
30
31 constexpr bool IsEOLChar(int ch) noexcept {
32 return ch == '\r' || ch == '\n';
33 }
34
35 constexpr bool IsIdentifierChar(int ch) noexcept {
36 return IsAlphaNumeric(ch) || ch == '_';
37 }
38
39 constexpr bool IsNumberContinue(int chPrev, int ch, int chNext) noexcept {
40 return ((ch == '+' || ch == '-') && (chPrev == 'e' || chPrev == 'E'))
41 || (ch == '.' && chNext != '.');
42 }
43
44 constexpr bool IsDecimalNumber(int chPrev, int ch, int chNext) noexcept {
45 return IsIdentifierChar(ch) || IsNumberContinue(chPrev, ch, chNext);
46 }
47
48 constexpr bool IsISODateTime(int ch, int chNext) noexcept {
49 return ((ch == '+' || ch == '-' || ch == ':' || ch == '.') && IsADigit(chNext))
50 || (ch == ' ' && (chNext == '+' || chNext == '-' || IsADigit(chNext)));
51 }
52
53 struct EscapeSequence {
54 int outerState = SCE_TOML_DEFAULT;
55 int digitsLeft = 0;
56
57 // highlight any character as escape sequence.
58 bool resetEscapeState(int state, int chNext) noexcept {
59 if (IsEOLChar(chNext)) {
60 return false;
61 }
62 outerState = state;
63 digitsLeft = 1;
64 if (chNext == 'x') {
65 digitsLeft = 3;
66 } else if (chNext == 'u') {
67 digitsLeft = 5;
68 } else if (chNext == 'U') {
69 digitsLeft = 9;
70 }
71 return true;
72 }
73 bool atEscapeEnd(int ch) noexcept {
74 --digitsLeft;
75 return digitsLeft <= 0 || !IsAHeXDigit(ch);
76 }
77 };
78
79 constexpr bool IsTripleString(int state) noexcept {
80 return state == SCE_TOML_TRIPLE_STRING_SQ || state == SCE_TOML_TRIPLE_STRING_DQ;
81 }
82
83 constexpr bool IsDoubleQuoted(int state) noexcept {
84 return state == SCE_TOML_STRING_DQ || state == SCE_TOML_TRIPLE_STRING_DQ;
85 }
86
87 constexpr int GetStringQuote(int state) noexcept {
88 return IsDoubleQuoted(state) ? '\"' : '\'';
89 }
90
91 constexpr bool IsTOMLOperator(int ch) noexcept {
92 return AnyOf(ch, '[', ']', '{', '}', ',', '=', '.', '+', '-');
93 }
94
95 constexpr bool IsTOMLUnquotedKey(int ch) noexcept {
96 return IsIdentifierChar(ch) || ch == '-';
97 }
98
99 constexpr bool IsWhiteSpace(int ch) noexcept {
100 return (ch == ' ') || ((ch >= 0x09) && (ch <= 0x0d));
101 }
102
103 int GetLineNextChar(StyleContext& sc) {
104 if (!IsWhiteSpace(sc.ch)) {
105 return sc.ch;
106 }
107 if (static_cast<Sci_Position>(sc.currentPos) + 1 == sc.lineStartNext) {
108 return '\0';
109 }
110 if (!IsWhiteSpace(sc.chNext)) {
111 return sc.chNext;
112 }
113 for (Sci_Position pos = 2; pos < sc.lineStartNext; pos++) {
114 const unsigned char chPos = sc.GetRelative(pos);
115 if (!IsWhiteSpace(chPos)) {
116 return chPos;
117 }
118 }
119 return '\0';
120 }
121
122 bool IsTOMLKey(StyleContext& sc, int braceCount, const WordList *kwList) {
123 if (braceCount) {
124 const int chNext = GetLineNextChar(sc);
125 if (chNext == '=' || chNext == '.' || chNext == '-') {
126 sc.ChangeState(SCE_TOML_KEY);
127 return true;
128 }
129 }
130 if (sc.state == SCE_TOML_IDENTIFIER) {
131 char s[8];
132 sc.GetCurrentLowered(s, sizeof(s));
133 #if defined(__clang__)
134 __builtin_assume(kwList != nullptr); // suppress [clang-analyzer-core.CallAndMessage]
135 #endif
136 if (kwList->InList(s)) {
137 sc.ChangeState(SCE_TOML_KEYWORD);
138 }
139 }
140 sc.SetState(SCE_TOML_DEFAULT);
141 return false;
142 }
143
144 enum class TOMLLineType {
145 None = 0,
146 Table,
147 CommentLine,
148 };
149
150 enum class TOMLKeyState {
151 Unquoted = 0,
152 Literal, // single-quoted
153 Quoted, // double-quoted
154 End,
155 };
156
157 void ColouriseTOMLDoc(Sci_PositionU startPos, Sci_Position lengthDoc, int initStyle, WordList *keywordLists[], Accessor &styler) {
158 int visibleChars = 0;
159 int chPrevNonWhite = 0;
160 int tableLevel = 0;
161 int braceCount = 0;
162 TOMLLineType lineType = TOMLLineType::None;
163 TOMLKeyState keyState = TOMLKeyState::Unquoted;
164 EscapeSequence escSeq;
165
166 if (initStyle == SCE_TOML_STRINGEOL) {
167 initStyle = SCE_TOML_DEFAULT;
168 }
169
170 StyleContext sc(startPos, lengthDoc, initStyle, styler);
171 if (sc.currentLine > 0) {
172 const int lineState = styler.GetLineState(sc.currentLine - 1);
173 /*
174 2: lineType
175 8: tableLevel
176 8: braceCount
177 */
178 braceCount = (lineState >> 10) & 0xff;
179 }
180
181 while (sc.More()) {
182 switch (sc.state) {
183 case SCE_TOML_OPERATOR:
184 sc.SetState(SCE_TOML_DEFAULT);
185 break;
186
187 case SCE_TOML_NUMBER:
188 if (!IsDecimalNumber(sc.chPrev, sc.ch, sc.chNext)) {
189 if (IsISODateTime(sc.ch, sc.chNext)) {
190 sc.ChangeState(SCE_TOML_DATETIME);
191 } else if (IsTOMLKey(sc, braceCount, nullptr)) {
192 keyState = TOMLKeyState::Unquoted;
193 continue;
194 }
195 }
196 break;
197
198 case SCE_TOML_DATETIME:
199 if (!(IsIdentifierChar(sc.ch) || IsISODateTime(sc.ch, sc.chNext))) {
200 if (IsTOMLKey(sc, braceCount, nullptr)) {
201 keyState = TOMLKeyState::Unquoted;
202 continue;
203 }
204 }
205 break;
206
207 case SCE_TOML_IDENTIFIER:
208 if (!IsIdentifierChar(sc.ch)) {
209 if (IsTOMLKey(sc, braceCount, keywordLists[0])) {
210 keyState = TOMLKeyState::Unquoted;
211 continue;
212 }
213 }
214 break;
215
216 case SCE_TOML_TABLE:
217 case SCE_TOML_KEY:
218 if (sc.atLineStart) {
219 sc.SetState(SCE_TOML_DEFAULT);
220 } else {
221 switch (keyState) {
222 case TOMLKeyState::Literal:
223 if (sc.ch == '\'') {
224 keyState = TOMLKeyState::Unquoted;
225 sc.Forward();
226 }
227 break;
228 case TOMLKeyState::Quoted:
229 if (sc.ch == '\\') {
230 sc.Forward();
231 } else if (sc.ch == '\"') {
232 keyState = TOMLKeyState::Unquoted;
233 sc.Forward();
234 }
235 break;
236 default:
237 break;
238 }
239 if (keyState == TOMLKeyState::Unquoted) {
240 if (sc.ch == '\'') {
241 keyState = TOMLKeyState::Literal;
242 } else if (sc.ch == '\"') {
243 keyState = TOMLKeyState::Quoted;
244 } else if (sc.ch == '.') {
245 if (sc.state == SCE_TOML_TABLE) {
246 ++tableLevel;
247 } else {
248 chPrevNonWhite = '.';
249 sc.SetState(SCE_TOML_OPERATOR);
250 sc.ForwardSetState(SCE_TOML_KEY);
251 // TODO: skip space after dot
252 continue;
253 }
254 } else if (sc.state == SCE_TOML_TABLE && sc.ch == ']') {
255 keyState = TOMLKeyState::End;
256 sc.Forward();
257 if (sc.ch == ']') {
258 sc.Forward();
259 }
260 const int chNext = GetLineNextChar(sc);
261 if (chNext == '#') {
262 sc.SetState(SCE_TOML_DEFAULT);
263 }
264 } else if (sc.state == SCE_TOML_KEY && !IsTOMLUnquotedKey(sc.ch)) {
265 const int chNext = GetLineNextChar(sc);
266 if (chNext == '=' || (chNext != '.' && chPrevNonWhite != '.')) {
267 keyState = TOMLKeyState::End;
268 sc.SetState(SCE_TOML_DEFAULT);
269 }
270 }
271 }
272 }
273 break;
274
275 case SCE_TOML_STRING_SQ:
276 case SCE_TOML_STRING_DQ:
277 case SCE_TOML_TRIPLE_STRING_SQ:
278 case SCE_TOML_TRIPLE_STRING_DQ:
279 if (sc.atLineStart && !IsTripleString(sc.state)) {
280 sc.SetState(SCE_TOML_DEFAULT);
281 } else if (sc.atLineEnd && !IsTripleString(sc.state)) {
282 sc.ChangeState(SCE_TOML_STRINGEOL);
283 } else if (sc.ch == '\\' && IsDoubleQuoted(sc.state)) {
284 if (escSeq.resetEscapeState(sc.state, sc.chNext)) {
285 sc.SetState(SCE_TOML_ESCAPECHAR);
286 sc.Forward();
287 }
288 } else if (sc.ch == GetStringQuote(sc.state) &&
289 (!IsTripleString(sc.state) || (sc.Match(IsDoubleQuoted(sc.state) ? R"(""")" : R"(''')")))) {
290 while (sc.ch == sc.chNext) {
291 sc.Forward();
292 }
293 sc.Forward();
294 if (!IsTripleString(sc.state) && IsTOMLKey(sc, braceCount, nullptr)) {
295 keyState = TOMLKeyState::Unquoted;
296 continue;
297 }
298 sc.SetState(SCE_TOML_DEFAULT);
299 }
300 break;
301
302 case SCE_TOML_STRINGEOL:
303 if (sc.atLineStart) {
304 sc.SetState(SCE_TOML_DEFAULT);
305 }
306 break;
307
308 case SCE_TOML_ESCAPECHAR:
309 if (escSeq.atEscapeEnd(sc.ch)) {
310 sc.SetState(escSeq.outerState);
311 continue;
312 }
313 break;
314
315 case SCE_TOML_ERROR:
316 if (sc.atLineStart) {
317 sc.SetState(SCE_TOML_DEFAULT);
318 } else if (sc.ch == '#') {
319 sc.SetState(SCE_TOML_COMMENT);
320 }
321 break;
322
323 case SCE_TOML_COMMENT:
324 if (sc.atLineStart) {
325 sc.SetState(SCE_TOML_DEFAULT);
326 }
327 break;
328 }
329
330 if (sc.state == SCE_TOML_DEFAULT) {
331 if (sc.ch == '#') {
332 sc.SetState(SCE_TOML_COMMENT);
333 if (visibleChars == 0) {
334 lineType = TOMLLineType::CommentLine;
335 }
336 } else if (visibleChars == 0 && braceCount == 0) {
337 if (sc.ch == '[') {
338 tableLevel = 0;
339 sc.SetState(SCE_TOML_TABLE);
340 if (sc.chNext == '[') {
341 sc.Forward();
342 }
343 keyState = TOMLKeyState::Unquoted;
344 lineType = TOMLLineType::Table;
345 } else if (sc.ch == '\'' || sc.ch == '\"') {
346 keyState = (sc.ch == '\'')? TOMLKeyState::Literal : TOMLKeyState::Quoted;
347 sc.SetState(SCE_TOML_KEY);
348 } else if (IsTOMLUnquotedKey(sc.ch)) {
349 keyState = TOMLKeyState::Unquoted;
350 sc.SetState(SCE_TOML_KEY);
351 } else if (!isspacechar(sc.ch)) {
352 // each line must be: key = value
353 sc.SetState(SCE_TOML_ERROR);
354 }
355 } else {
356 if (sc.ch == '\'') {
357 if (sc.Match(R"(''')")) {
358 sc.SetState(SCE_TOML_TRIPLE_STRING_SQ);
359 sc.Forward(2);
360 } else {
361 sc.SetState(SCE_TOML_STRING_SQ);
362 }
363 } else if (sc.ch == '"') {
364 if (sc.Match(R"(""")")) {
365 sc.SetState(SCE_TOML_TRIPLE_STRING_DQ);
366 sc.Forward(2);
367 } else {
368 sc.SetState(SCE_TOML_STRING_DQ);
369 }
370 } else if (IsADigit(sc.ch)) {
371 sc.SetState(SCE_TOML_NUMBER);
372 } else if (IsLowerCase(sc.ch)) {
373 sc.SetState(SCE_TOML_IDENTIFIER);
374 } else if (IsTOMLOperator(sc.ch)) {
375 sc.SetState(SCE_TOML_OPERATOR);
376 if (sc.ch == '[' || sc.ch == '{') {
377 ++braceCount;
378 } else if (sc.ch == ']' || sc.ch == '}') {
379 if (braceCount > 0) {
380 --braceCount;
381 }
382 }
383 } else if (braceCount && IsTOMLUnquotedKey(sc.ch)) {
384 // Inline Table
385 keyState = TOMLKeyState::Unquoted;
386 sc.SetState(SCE_TOML_KEY);
387 }
388 }
389 }
390
391 if (!isspacechar(sc.ch)) {
392 chPrevNonWhite = sc.ch;
393 ++visibleChars;
394 }
395 if (sc.atLineEnd) {
396 const int lineState = (tableLevel << 2) | (braceCount << 10) | static_cast<int>(lineType);
397 styler.SetLineState(sc.currentLine, lineState);
398 lineType = TOMLLineType::None;
399 visibleChars = 0;
400 chPrevNonWhite = 0;
401 tableLevel = 0;
402 keyState = TOMLKeyState::Unquoted;
403 }
404 sc.Forward();
405 }
406
407 sc.Complete();
408 }
409
410 constexpr TOMLLineType GetLineType(int lineState) noexcept {
411 return static_cast<TOMLLineType>(lineState & 3);
412 }
413
414 constexpr int GetTableLevel(int lineState) noexcept {
415 return (lineState >> 2) & 0xff;
416 }
417
418 // code folding based on LexProps
419 void FoldTOMLDoc(Sci_PositionU startPos, Sci_Position lengthDoc, int /*initStyle*/, WordList *[] /*keywordLists*/, Accessor &styler) {
420 const Sci_Position endPos = startPos + lengthDoc;
421 const Sci_Position maxLines = styler.GetLine((endPos == styler.Length()) ? endPos : endPos - 1);
422
423 Sci_Position lineCurrent = styler.GetLine(startPos);
424
425 int prevLevel = SC_FOLDLEVELBASE;
426 TOMLLineType prevType = TOMLLineType::None;
427 TOMLLineType prev2Type = TOMLLineType::None;
428 if (lineCurrent > 0) {
429 prevLevel = styler.LevelAt(lineCurrent - 1);
430 prevType = GetLineType(styler.GetLineState(lineCurrent - 1));
431 if (lineCurrent >= 2) {
432 prev2Type = GetLineType(styler.GetLineState(lineCurrent - 2));
433 }
434 }
435
436 bool commentHead = (prevType == TOMLLineType::CommentLine) && (prevLevel & SC_FOLDLEVELHEADERFLAG);
437 while (lineCurrent <= maxLines) {
438 int nextLevel;
439 const int lineState = styler.GetLineState(lineCurrent);
440 const TOMLLineType lineType = GetLineType(lineState);
441
442 if (lineType == TOMLLineType::CommentLine) {
443 if (prevLevel & SC_FOLDLEVELHEADERFLAG) {
444 nextLevel = (prevLevel & SC_FOLDLEVELNUMBERMASK) + 1;
445 } else {
446 nextLevel = prevLevel;
447 }
448 commentHead = prevType != TOMLLineType::CommentLine;
449 nextLevel |= commentHead ? SC_FOLDLEVELHEADERFLAG : 0;
450 } else {
451 if (lineType == TOMLLineType::Table) {
452 nextLevel = SC_FOLDLEVELBASE + GetTableLevel(lineState);
453 if ((prevType == TOMLLineType::CommentLine) && prevLevel <= nextLevel) {
454 // comment above nested table
455 commentHead = true;
456 prevLevel = nextLevel - 1;
457 } else if ((prevType == TOMLLineType::Table) && (prevLevel & SC_FOLDLEVELNUMBERMASK) >= nextLevel) {
458 commentHead = true; // empty table
459 }
460 nextLevel |= SC_FOLDLEVELHEADERFLAG;
461 } else {
462 if (commentHead) {
463 nextLevel = prevLevel & SC_FOLDLEVELNUMBERMASK;
464 } else if (prevLevel & SC_FOLDLEVELHEADERFLAG) {
465 nextLevel = (prevLevel & SC_FOLDLEVELNUMBERMASK) + 1;
466 } else if ((prevType == TOMLLineType::CommentLine) && (prev2Type == TOMLLineType::CommentLine)) {
467 nextLevel = prevLevel - 1;
468 } else {
469 nextLevel = prevLevel;
470 }
471 }
472
473 if (commentHead) {
474 commentHead = false;
475 styler.SetLevel(lineCurrent - 1, prevLevel & SC_FOLDLEVELNUMBERMASK);
476 }
477 }
478
479 styler.SetLevel(lineCurrent, nextLevel);
480 prevLevel = nextLevel;
481 prev2Type = prevType;
482 prevType = lineType;
483 lineCurrent++;
484 }
485 }
486
487 } // unnamed namespace end
488
489 static const char *const tomlWordListDesc[] = {
490 "Keywords",
491 0
492 };
493
494 extern const LexerModule lmTOML(SCLEX_TOML, ColouriseTOMLDoc, "toml", FoldTOMLDoc, tomlWordListDesc);