Debian Codegraph @

scintilla/lexilla/lexers/LexRaku.cxx

raw
1 /** @file LexRaku.cxx
2 ** Lexer for Raku
3 **
4 ** Copyright (c) 2019 Mark Reay <[email protected]>
5 **/
6 // Copyright 1998-2005 by Neil Hodgson <[email protected]>
7 // The License.txt file describes the conditions under which this software may be distributed.
8
9 /*
10 * Raku (Perl6) Lexer for Scintilla
11 * ---------------------------------
12 * ---------------------------------
13 * 06-Dec-2019: More Unicode support:
14 * - Added a full scope of allowed numbers and letters
15 * 29-Nov-2019: More highlighting / implemented basic folding:
16 * - Operators (blanket cover, no sequence checking)
17 * - Class / Grammar name highlighting
18 * - Folding:
19 * - Comments: line / multi-line
20 * - POD sections
21 * - Code blocks {}
22 * 26-Nov-2019: Basic syntax highlighting covering the following:
23 * - Comments, both line and embedded (multi-line)
24 * - POD, no inline highlighting as yet...
25 * - Heredoc block string, with variable highlighting (with qq)
26 * - Strings, with variable highlighting (with ")
27 * - Q Language, including adverbs (also basic q and qq)
28 * - Regex, including adverbs
29 * - Numbers
30 * - Bareword / identifiers
31 * - Types
32 * - Variables: mu, positional, associative, callable
33 * TODO:
34 * - POD inline
35 * - Better operator sequence coverage
36 */
37
38 #include <stdlib.h>
39 #include <string.h>
40 #include <stdio.h>
41 #include <stdarg.h>
42 #include <assert.h>
43 #include <ctype.h>
44
45 #include <string>
46 #include <string_view>
47 #include <vector>
48 #include <map>
49 #include <functional>
50
51 #include "ILexer.h"
52 #include "Scintilla.h"
53 #include "SciLexer.h"
54
55 #include "WordList.h"
56 #include "LexAccessor.h"
57 #include "StyleContext.h"
58 #include "CharacterSet.h"
59 #include "CharacterCategory.h"
60 #include "LexerModule.h"
61 #include "OptionSet.h"
62 #include "DefaultLexer.h"
63
64 using namespace Scintilla;
65 using namespace Lexilla;
66
67 namespace { // anonymous namespace to isolate any name clashes
68 /*----------------------------------------------------------------------------*
69 * --- DEFINITIONS: OPTIONS / CONSTANTS ---
70 *----------------------------------------------------------------------------*/
71
72 // Number types
73 #define RAKUNUM_BINARY 1 // order is significant: 1-3 cannot have a dot
74 #define RAKUNUM_OCTAL 2
75 #define RAKUNUM_FLOAT_EXP 3 // exponent part only
76 #define RAKUNUM_HEX 4 // may be a hex float
77 #define RAKUNUM_DECIMAL 5 // 1-5 are numbers; 6-7 are strings
78 #define RAKUNUM_VECTOR 6
79 #define RAKUNUM_V_VECTOR 7
80 #define RAKUNUM_VERSION 8 // can contain multiple '.'s
81 #define RAKUNUM_BAD 9
82
83 // Regex / Q string types
84 #define RAKUTYPE_REGEX_NORM 0 // 0 char ident
85 #define RAKUTYPE_REGEX_S 1 // order is significant:
86 #define RAKUTYPE_REGEX_M 2 // 1 char ident
87 #define RAKUTYPE_REGEX_Y 3 // 1 char ident
88 #define RAKUTYPE_REGEX 4 // > RAKUTYPE_REGEX == 2 char identifiers
89 #define RAKUTYPE_REGEX_RX 5 // 2 char ident
90 #define RAKUTYPE_REGEX_TR 6 // 2 char ident
91 #define RAKUTYPE_QLANG 7 // < RAKUTYPE_QLANG == RAKUTYPE_REGEX_?
92 #define RAKUTYPE_STR_WQ 8 // 0 char ident < word quote >
93 #define RAKUTYPE_STR_Q 9 // 1 char ident
94 #define RAKUTYPE_STR_QX 10 // 2 char ident
95 #define RAKUTYPE_STR_QW 11 // 2 char ident
96 #define RAKUTYPE_STR_QQ 12 // 2 char ident
97 #define RAKUTYPE_STR_QQX 13 // 3 char ident
98 #define RAKUTYPE_STR_QQW 14 // 3 char ident
99 #define RAKUTYPE_STR_QQWW 15 // 4 char ident
100
101 // Delimiter types
102 #define RAKUDELIM_BRACKET 0 // bracket: regex, Q language
103 #define RAKUDELIM_QUOTE 1 // quote: normal string
104
105 // rakuWordLists: keywords as defined in config
106 const char *const rakuWordLists[] = {
107 "Keywords and identifiers",
108 "Functions",
109 "Types basic",
110 "Types composite",
111 "Types domain-specific",
112 "Types exception",
113 "Adverbs",
114 nullptr,
115 };
116
117 // Options and defaults
118 struct OptionsRaku {
119 bool fold;
120 bool foldCompact;
121 bool foldComment;
122 bool foldCommentMultiline;
123 bool foldCommentPOD;
124 OptionsRaku() {
125 fold = true;
126 foldCompact = false;
127 foldComment = true;
128 foldCommentMultiline = true;
129 foldCommentPOD = true;
130 }
131 };
132
133 // init options and words
134 struct OptionSetRaku : public OptionSet<OptionsRaku> {
135 OptionSetRaku() {
136 DefineProperty("fold", &OptionsRaku::fold);
137 DefineProperty("fold.comment", &OptionsRaku::foldComment);
138 DefineProperty("fold.compact", &OptionsRaku::foldCompact);
139
140 DefineProperty("fold.raku.comment.multiline", &OptionsRaku::foldCommentMultiline,
141 "Set this property to 0 to disable folding multi-line comments when fold.comment=1.");
142 DefineProperty("fold.raku.comment.pod", &OptionsRaku::foldCommentPOD,
143 "Set this property to 0 to disable folding POD comments when fold.comment=1.");
144
145 // init word lists
146 DefineWordListSets(rakuWordLists);
147 }
148 };
149
150 // Delimiter pair
151 struct DelimPair {
152 int opener; // opener char
153 int closer[2]; // closer chars
154 bool interpol; // can variables be interpolated?
155 short count; // delimiter char count
156 DelimPair() {
157 opener = 0;
158 closer[0] = 0;
159 closer[1] = 0;
160 interpol = false;
161 count = 0;
162 }
163 bool isCloser(int ch) const {
164 return ch == closer[0] || ch == closer[1];
165 }
166 };
167
168 /*----------------------------------------------------------------------------*
169 * --- FUNCTIONS ---
170 *----------------------------------------------------------------------------*/
171
172 /*
173 * IsANewLine
174 * - returns true if this is a new line char
175 */
176 constexpr bool IsANewLine(int ch) noexcept {
177 return ch == '\r' || ch == '\n';
178 }
179
180 /*
181 * IsAWhitespace
182 * - returns true if this is a whitespace (or newline) char
183 */
184 bool IsAWhitespace(int ch) noexcept {
185 return IsASpaceOrTab(ch) || IsANewLine(ch);
186 }
187
188 /*
189 * IsAlphabet
190 * - returns true if this is an alphabetical char
191 */
192 constexpr bool IsAlphabet(int ch) noexcept {
193 return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z');
194 }
195
196 /*
197 * IsCommentLine
198 * - returns true if this is a comment line
199 * - tests: SCE_RAKU_COMMENTLINE or SCE_RAKU_COMMENTEMBED
200 * modified from: LexPerl.cxx
201 */
202 bool IsCommentLine(Sci_Position line, LexAccessor &styler, int type = SCE_RAKU_COMMENTLINE) {
203 Sci_Position pos = styler.LineStart(line);
204 Sci_Position eol_pos = styler.LineStart(line + 1) - 1;
205 for (Sci_Position i = pos; i < eol_pos; i++) {
206 char ch = styler[i];
207 int style = styler.StyleAt(i);
208 if (type == SCE_RAKU_COMMENTEMBED) {
209 if (i == (eol_pos - 1) && style == type)
210 return true;
211 } else { // make sure the line is NOT a SCE_RAKU_COMMENTEMBED
212 if (ch == '#' && style == type && styler[i+1] != '`' )
213 return true;
214 else if (!IsASpaceOrTab(ch))
215 return false;
216 }
217 }
218 return false;
219 }
220
221 /*
222 * ContainsQTo
223 * - returns true if this range contains ":to" in style SCE_RAKU_ADVERB indicating the start
224 * of a SCE_RAKU_HEREDOC_Q or SCE_RAKU_HEREDOC_QQ.
225 */
226 bool ContainsQTo(Sci_Position start, Sci_Position end, LexAccessor &styler) {
227 std::string adverb;
228 for (Sci_Position i = start; i < end; i++) {
229 if (styler.StyleAt(i) == SCE_RAKU_ADVERB) {
230 adverb.push_back(styler[i]);
231 }
232 }
233 return adverb.find(":to") != std::string::npos;
234 }
235
236 /*
237 * GetBracketCloseChar
238 * - returns the end bracket char: opposite of start
239 * - see: http://www.unicode.org/Public/5.1.0/ucd/BidiMirroring.txt (first section)
240 * - Categories are general matches for valid BiDi types
241 * - Most closer chars are opener + 1
242 */
243 int GetBracketCloseChar(const int ch) noexcept {
244 const CharacterCategory cc = CategoriseCharacter(ch);
245 switch (cc) {
246 case ccSm:
247 switch (ch) {
248 case 0x3C: return 0x3E; // LESS-THAN SIGN
249 case 0x2208: return 0x220B; // ELEMENT OF
250 case 0x2209: return 0x220C; // NOT AN ELEMENT OF
251 case 0x220A: return 0x220D; // SMALL ELEMENT OF
252 case 0x2215: return 0x29F5; // DIVISION SLASH
253 case 0x2243: return 0x22CD; // ASYMPTOTICALLY EQUAL TO
254 case 0x2298: return 0x29B8; // CIRCLED DIVISION SLASH
255 case 0x22A6: return 0x2ADE; // ASSERTION
256 case 0x22A8: return 0x2AE4; // TRUE
257 case 0x22A9: return 0x2AE3; // FORCES
258 case 0x22AB: return 0x2AE5; // DOUBLE VERTICAL BAR DOUBLE RIGHT TURNSTILE
259 case 0x22F2: return 0x22FA; // ELEMENT OF WITH LONG HORIZONTAL STROKE
260 case 0x22F3: return 0x22FB; // ELEMENT OF WITH VERTICAL BAR AT END OF HORIZONTAL STROKE
261 case 0x22F4: return 0x22FC; // SMALL ELEMENT OF WITH VERTICAL BAR AT END OF HORIZONTAL STROKE
262 case 0x22F6: return 0x22FD; // ELEMENT OF WITH OVERBAR
263 case 0x22F7: return 0x22FE; // SMALL ELEMENT OF WITH OVERBAR
264 case 0xFF1C: return 0xFF1E; // FULLWIDTH LESS-THAN SIGN
265 }
266 break;
267 case ccPs:
268 switch (ch) {
269 case 0x5B: return 0x5D; // LEFT SQUARE BRACKET
270 case 0x7B: return 0x7D; // LEFT CURLY BRACKET
271 case 0x298D: return 0x2990; // LEFT SQUARE BRACKET WITH TICK IN TOP CORNER
272 case 0x298F: return 0x298E; // LEFT SQUARE BRACKET WITH TICK IN BOTTOM CORNER
273 case 0xFF3B: return 0xFF3D; // FULLWIDTH LEFT SQUARE BRACKET
274 case 0xFF5B: return 0xFF5D; // FULLWIDTH LEFT CURLY BRACKET
275 }
276 break;
277 case ccPi:
278 break;
279 default: return 0;
280 }
281 return ch + 1;
282 }
283
284 /*
285 * IsValidQuoteOpener
286 * -
287 */
288 bool IsValidQuoteOpener(const int ch, DelimPair &dp, int type = RAKUDELIM_BRACKET) noexcept {
289 dp.closer[0] = 0;
290 dp.closer[1] = 0;
291 dp.interpol = true;
292 if (type == RAKUDELIM_QUOTE) {
293 switch (ch) {
294 // Opener Closer Description
295 case '\'': dp.closer[0] = '\''; // APOSTROPHE
296 dp.interpol = false;
297 break;
298 case '"': dp.closer[0] = '"'; // QUOTATION MARK
299 break;
300 case 0x2018: dp.closer[0] = 0x2019; // LEFT SINGLE QUOTATION MARK
301 dp.interpol = false;
302 break;
303 case 0x201C: dp.closer[0] = 0x201D; // LEFT DOUBLE QUOTATION MARK
304 break;
305 case 0x201D: dp.closer[0] = 0x201C; // RIGHT DOUBLE QUOTATION MARK
306 break;
307 case 0x201E: dp.closer[0] = 0x201C; // DOUBLE LOW-9 QUOTATION MARK
308 dp.closer[1] = 0x201D;
309 break;
310 case 0xFF62: dp.closer[0] = 0xFF63; // HALFWIDTH LEFT CORNER BRACKET
311 dp.interpol = false;
312 break;
313 default: return false;
314 }
315 } else if (type == RAKUDELIM_BRACKET) {
316 dp.closer[0] = GetBracketCloseChar(ch);
317 }
318 dp.opener = ch;
319 dp.count = 1;
320 return dp.closer[0] > 0;
321 }
322
323 /*
324 * IsBracketOpenChar
325 * - true if this is a valid start bracket character
326 */
327 bool IsBracketOpenChar(int ch) noexcept {
328 return GetBracketCloseChar(ch) > 0;
329 }
330
331 /*
332 * IsValidRegOrQAdjacent
333 * - returns true if ch is a valid character to put directly after Q / q
334 * * ref: Q Language: https://docs.raku.org/language/quoting
335 */
336 bool IsValidRegOrQAdjacent(int ch) noexcept {
337 return !(IsAlphaNumeric(ch) || ch == '_' || ch == '(' || ch == ')' || ch == '\'' );
338 }
339
340 /*
341 * IsValidRegOrQPrecede
342 * - returns true if ch is a valid preceding character to put directly before Q / q
343 * * ref: Q Language: https://docs.raku.org/language/quoting
344 */
345 bool IsValidRegOrQPrecede(int ch) noexcept {
346 return !(IsAlphaNumeric(ch) || ch == '_');
347 }
348
349 /*
350 * MatchCharInRange
351 * - returns true if the mach character is found in range (of length)
352 * - ignoreDelim (default false)
353 */
354 bool MatchCharInRange(StyleContext &sc, const Sci_Position length,
355 const int match, bool ignoreDelim = false) {
356 Sci_Position len = 0;
357 int chPrev = sc.chPrev;
358 while (++len < length) {
359 const int ch = sc.GetRelativeCharacter(len);
360 if (ch == match && (ignoreDelim || chPrev != '\\'))
361 return true;
362 }
363 return false;
364 }
365
366 /*
367 * PrevNonWhitespaceChar
368 * - returns the last non-whitespace char
369 */
370 int PrevNonWhitespaceChar(StyleContext &sc) {
371 Sci_Position rel = 0;
372 Sci_Position max_back = 0 - sc.currentPos;
373 while (--rel > max_back) {
374 const int ch = sc.GetRelativeCharacter(rel);
375 if (!IsAWhitespace(ch))
376 return ch;
377 }
378 return 0; // no matching char
379 }
380
381 /*
382 * IsQLangStartAtScPos
383 * - returns true if this is a valid Q Language sc position
384 * - ref: https://docs.raku.org/language/quoting
385 * - Q :adverb :adverb //;
386 * - q,qx,qw,qq,qqx,qqw,qqww :adverb /:adverb /;
387 */
388 bool IsQLangStartAtScPos(StyleContext &sc, int &type, const Sci_Position length) {
389 const bool valid_adj = IsValidRegOrQAdjacent(sc.chNext);
390 const int chFw2 = sc.GetRelativeCharacter(2);
391 const int chFw3 = sc.GetRelativeCharacter(3);
392 type = -1;
393 if (IsValidRegOrQPrecede(sc.chPrev)) {
394 if (sc.ch == 'Q' && valid_adj) {
395 type = RAKUTYPE_QLANG;
396 } else if (sc.ch == 'q') {
397 switch (sc.chNext) {
398 case 'x':
399 type = RAKUTYPE_STR_QX;
400 break;
401 case 'w':
402 type = RAKUTYPE_STR_QW;
403 break;
404 case 'q':
405 if (chFw2 == 'x') {
406 type = RAKUTYPE_STR_QQX;
407 } else if (chFw2 == 'w') {
408 if (chFw3 == 'w') {
409 type = RAKUTYPE_STR_QQWW;
410 } else {
411 type = RAKUTYPE_STR_QQW;
412 }
413 } else {
414 type = RAKUTYPE_STR_QQ;
415 }
416 break;
417 default:
418 type = RAKUTYPE_STR_Q;
419 }
420 } else if (sc.ch == '<' && MatchCharInRange(sc, length, '>')) {
421 type = RAKUTYPE_STR_WQ; // < word quote >
422 }
423 }
424 return type >= 0;
425 }
426
427 /*
428 * IsRegexStartAtScPos
429 * - returns true if this is a valid Regex sc position
430 * - ref: https://docs.raku.org/language/regexes
431 * - Regex: (rx/s/m/tr/y) :adverb /:adverb /;
432 * - regex R :adverb //;
433 * - /:adverb /;
434 */
435 bool IsRegexStartAtScPos(StyleContext &sc, int &type, CharacterSet &set) {
436 const bool valid_adj = IsValidRegOrQAdjacent(sc.chNext);
437 type = -1;
438 if (IsValidRegOrQPrecede(sc.chPrev)) {
439 switch (sc.ch) {
440 case 'r':
441 if (sc.chNext == 'x')
442 type = RAKUTYPE_REGEX_RX;
443 break;
444 case 't':
445 case 'T':
446 if (sc.chNext == 'r' || sc.chNext == 'R')
447 type = RAKUTYPE_REGEX_TR;
448 break;
449 case 'm':
450 if (valid_adj)
451 type = RAKUTYPE_REGEX_M;
452 break;
453 case 's':
454 case 'S':
455 if (valid_adj)
456 type = RAKUTYPE_REGEX_S;
457 break;
458 case 'y':
459 if (valid_adj)
460 type = RAKUTYPE_REGEX_Y;
461 break;
462 case '/':
463 if (set.Contains(PrevNonWhitespaceChar(sc)))
464 type = RAKUTYPE_REGEX_NORM;
465 }
466 }
467 return type >= 0;
468 }
469
470 /*
471 * IsValidIdentPrecede
472 * - returns if ch is a valid preceding char to put directly before an identifier
473 */
474 bool IsValidIdentPrecede(int ch) noexcept {
475 return !(IsAlphaNumeric(ch) || ch == '_' || ch == '@' || ch == '$' || ch == '%');
476 }
477
478 /*
479 * IsValidDelimiter
480 * - returns if ch is a valid delimiter (most chars are valid)
481 * * ref: Q Language: https://docs.raku.org/language/quoting
482 */
483 bool IsValidDelimiter(int ch) noexcept {
484 return !(IsAlphaNumeric(ch) || ch == ':');
485 }
486
487 /*
488 * GetDelimiterCloseChar
489 * - returns the corresponding close char for a given delimiter (could be the same char)
490 */
491 int GetDelimiterCloseChar(int ch) noexcept {
492 int ch_end = GetBracketCloseChar(ch);
493 if (ch_end == 0 && IsValidDelimiter(ch)) {
494 ch_end = ch;
495 }
496 return ch_end;
497 }
498
499 /*
500 * GetRepeatCharCount
501 * - returns the occurrence count of match
502 */
503 Sci_Position GetRepeatCharCount(StyleContext &sc, int chMatch, Sci_Position length) {
504 Sci_Position cnt = 0;
505 while (cnt < length) {
506 if (sc.GetRelativeCharacter(cnt) != chMatch) {
507 break;
508 }
509 cnt++;
510 }
511 return cnt;
512 }
513
514 /*
515 * LengthToDelimiter
516 * - returns the length until the end of a delimited string section
517 * - Ignores nested delimiters (if opener != closer)
518 * - no trailing char after last closer (default false)
519 */
520 Sci_Position LengthToDelimiter(StyleContext &sc, const DelimPair &dp,
521 Sci_Position length, bool noTrailing = false) {
522 short cnt_open = 0; // count open bracket
523 short cnt_close = 0; // count close bracket
524 bool is_escape = false; // has been escaped using '\'?
525 Sci_Position len = 0; // count characters
526 int chOpener = dp.opener; // look for nested opener / closer
527 if (dp.opener == dp.closer[0])
528 chOpener = 0; // no opening delimiter (no nesting possible)
529
530 while (len < length) {
531 const int chPrev = sc.GetRelativeCharacter(len - 1);
532 const int ch = sc.GetRelativeCharacter(len);
533 const int chNext = sc.GetRelativeCharacter(len+1);
534
535 if (cnt_open == 0 && cnt_close == dp.count) {
536 return len; // end condition has been met
537 } else if (is_escape) {
538 is_escape = false;
539 } else if (ch == '\\') {
540 is_escape = true;
541 } else {
542 if (ch == chOpener) {
543 cnt_open++; // open nested bracket
544 } else if (dp.isCloser(ch)) {
545 if ( cnt_open > 0 ) {
546 cnt_open--; // close nested bracket
547 } else if (dp.count > 1 && cnt_close < (dp.count - 1)) {
548 if (cnt_close > 1) {
549 if (dp.isCloser(chPrev)) {
550 cnt_close++;
551 } else { // reset if previous char was not close
552 cnt_close = 0;
553 }
554 } else {
555 cnt_close++;
556 }
557 } else if (!noTrailing || (IsAWhitespace(chNext))) {
558 cnt_close++; // found last close
559 if (cnt_close > 1 && !dp.isCloser(chPrev)) {
560 cnt_close = 0; // reset if previous char was not close
561 }
562 } else {
563 cnt_close = 0; // non handled close: reset
564 }
565 } else if (IsANewLine(ch)) {
566 cnt_open = 0; // reset after each line
567 cnt_close = 0;
568 }
569 }
570 len++;
571 }
572 return -1; // end condition has NOT been met
573 }
574
575 /*
576 * LengthToEndHeredoc
577 * - returns the length until the end of a heredoc section
578 * - delimiter string MUST begin on a new line
579 */
580 Sci_Position LengthToEndHeredoc(const StyleContext &sc, LexAccessor &styler,
581 const Sci_Position length, const char *delim) {
582 bool on_new_ln = false;
583 int i = 0; // str index
584 for (int n = 0; n < length; n++) {
585 const char ch = styler.SafeGetCharAt(sc.currentPos + n, 0);
586 if (on_new_ln) {
587 if (delim[i] == '\0')
588 return n; // at end of str, match found!
589 if (ch != delim[i++])
590 i = 0; // no char match, reset 'i'ndex
591 }
592 if (i == 0) // detect new line
593 on_new_ln = IsANewLine(ch);
594 }
595 return -1; // no match found
596 }
597
598 /*
599 * LengthToNextChar
600 * - returns the length until the next character
601 */
602 Sci_Position LengthToNextChar(StyleContext &sc, const Sci_Position length) {
603 Sci_Position len = 0;
604 while (++len < length) {
605 const int ch = sc.GetRelativeCharacter(len);
606 if (!IsASpaceOrTab(ch) && !IsANewLine(ch)) {
607 break;
608 }
609 }
610 return len;
611 }
612
613 /*
614 * GetRelativeString
615 * - gets a relative string and sets it in &str
616 * - resets string before setting
617 */
618 void GetRelativeString(StyleContext &sc, Sci_Position offset, Sci_Position length,
619 std::string &str) {
620 Sci_Position pos = offset;
621 str.clear();
622 while (pos < length) {
623 str += sc.GetRelativeCharacter(pos++);
624 }
625 }
626
627 } // end anonymous namespace
628
629 /*----------------------------------------------------------------------------*
630 * --- class: LexerRaku ---
631 *----------------------------------------------------------------------------*/
632 //class LexerRaku : public ILexerWithMetaData {
633 class LexerRaku : public DefaultLexer {
634 CharacterSet setWord;
635 CharacterSet setSigil;
636 CharacterSet setTwigil;
637 CharacterSet setOperator;
638 CharacterSet setSpecialVar;
639 WordList regexIdent; // identifiers that specify a regex
640 OptionsRaku options; // Options from config
641 OptionSetRaku osRaku;
642 WordList keywords; // Word Lists from config
643 WordList functions;
644 WordList typesBasic;
645 WordList typesComposite;
646 WordList typesDomainSpecific;
647 WordList typesExceptions;
648 WordList adverbs;
649
650 public:
651 // Defined as explicit, so that constructor can not be copied
652 explicit LexerRaku() :
653 DefaultLexer("raku", SCLEX_RAKU),
654 setWord(CharacterSet::setAlphaNum, "-_", 0x80),
655 setSigil(CharacterSet::setNone, "$&%@"),
656 setTwigil(CharacterSet::setNone, "!*.:<=?^~"),
657 setOperator(CharacterSet::setNone, "^&\\()-+=|{}[]:;<>,?!.~"),
658 setSpecialVar(CharacterSet::setNone, "_/!") {
659 regexIdent.Set("regex rule token");
660 }
661 // Deleted so LexerRaku objects can not be copied.
662 LexerRaku(const LexerRaku &) = delete;
663 LexerRaku(LexerRaku &&) = delete;
664 void operator=(const LexerRaku &) = delete;
665 void operator=(LexerRaku &&) = delete;
666 virtual ~LexerRaku() {
667 }
668 void SCI_METHOD Release() noexcept override {
669 delete this;
670 }
671 int SCI_METHOD Version() const noexcept override {
672 return lvRelease5;
673 }
674 const char *SCI_METHOD PropertyNames() override {
675 return osRaku.PropertyNames();
676 }
677 int SCI_METHOD PropertyType(const char *name) override {
678 return osRaku.PropertyType(name);
679 }
680 const char *SCI_METHOD DescribeProperty(const char *name) override {
681 return osRaku.DescribeProperty(name);
682 }
683 Sci_Position SCI_METHOD PropertySet(const char *key, const char *val) override;
684 const char *SCI_METHOD PropertyGet(const char *key) override {
685 return osRaku.PropertyGet(key);
686 }
687 const char *SCI_METHOD DescribeWordListSets() override {
688 return osRaku.DescribeWordListSets();
689 }
690 Sci_Position SCI_METHOD WordListSet(int n, const char *wl) override;
691 void SCI_METHOD Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override;
692 void SCI_METHOD Fold(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override;
693
694 static ILexer5 *LexerFactoryRaku() {
695 return new LexerRaku();
696 }
697
698 protected:
699 bool IsOperatorChar(const int ch);
700 bool IsWordChar(const int ch, bool allowNumber = true);
701 bool IsWordStartChar(const int ch);
702 bool IsNumberChar(const int ch, int base = 10);
703 bool ProcessRegexTwinCapture(StyleContext &sc, const Sci_Position length,
704 int &type, const DelimPair &dp);
705 void ProcessStringVars(StyleContext &sc, const Sci_Position length, const int varState);
706 bool ProcessValidRegQlangStart(StyleContext &sc, Sci_Position length, const int type,
707 WordList &wordsAdverbs, DelimPair &dp);
708 Sci_Position LengthToNonWordChar(StyleContext &sc, Sci_Position length,
709 char *s, const int size, Sci_Position offset = 0);
710 };
711
712 /*----------------------------------------------------------------------------*
713 * --- METHODS: LexerRaku ---
714 *----------------------------------------------------------------------------*/
715
716 /*
717 * LexerRaku::IsOperatorChar
718 * - Test for both ASCII and Unicode operators
719 * see: https://docs.raku.org/language/unicode_entry
720 */
721 bool LexerRaku::IsOperatorChar(const int ch) {
722 if (ch > 0x7F) {
723 switch (ch) {
724 // Unicode ASCII Equiv.
725 case 0x2208: // (elem)
726 case 0x2209: // !(elem)
727 case 0x220B: // (cont)
728 case 0x220C: // !(cont)
729 case 0x2216: // (-)
730 case 0x2229: // (&)
731 case 0x222A: // (|)
732 case 0x2282: // (<)
733 case 0x2283: // (>)
734 case 0x2284: // !(<)
735 case 0x2285: // !(>)
736 case 0x2286: // (<=)
737 case 0x2287: // (>=)
738 case 0x2288: // !(<=)
739 case 0x2289: // !(>=)
740 case 0x228D: // (.)
741 case 0x228E: // (+)
742 case 0x2296: // (^)
743 return true;
744 }
745 }
746 return setOperator.Contains(ch);
747 }
748
749 /*
750 * LexerRaku::IsWordChar
751 * - Test for both ASCII and Unicode identifier characters
752 * see: https://docs.raku.org/language/unicode_ascii
753 * also: ftp://ftp.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt
754 * FIXME: *still* may not contain all valid characters
755 */
756 bool LexerRaku::IsWordChar(const int ch, bool allowNumber) {
757 // Unicode numbers should not appear in word identifiers
758 if (ch > 0x7F) {
759 const CharacterCategory cc = CategoriseCharacter(ch);
760 switch (cc) {
761 // Letters
762 case ccLu:
763 case ccLl:
764 case ccLt:
765 case ccLm:
766 case ccLo:
767 return true;
768 default:
769 return false;
770 }
771 } else if (allowNumber && IsADigit(ch)) {
772 return true; // an ASCII number type
773 }
774 return setWord.Contains(ch);
775 }
776
777 /*
778 * LexerRaku::IsWordStartChar
779 * - Test for both ASCII and Unicode identifier "start / first" characters
780 */
781 bool LexerRaku::IsWordStartChar(const int ch) {
782 return ch != '-' && IsWordChar(ch, false); // no numbers allowed
783 }
784
785 /*
786 * LexerRaku::IsNumberChar
787 * - Test for both ASCII and Unicode identifier number characters
788 * see: https://docs.raku.org/language/unicode_ascii
789 * also: ftp://ftp.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt
790 * FILTERED by Unicode letters that are NUMBER
791 * and NOT PARENTHESIZED or CIRCLED
792 * FIXME: *still* may not contain all valid number characters
793 */
794 bool LexerRaku::IsNumberChar(const int ch, int base) {
795 if (ch > 0x7F) {
796 const CharacterCategory cc = CategoriseCharacter(ch);
797 switch (cc) {
798 // Numbers
799 case ccNd:
800 case ccNl:
801 case ccNo:
802 return true;
803 default:
804 return false;
805 }
806 }
807 return IsADigit(ch, base);
808 }
809
810 /*
811 * LexerRaku::PropertySet
812 * -
813 */
814 Sci_Position SCI_METHOD LexerRaku::PropertySet(const char *key, const char *val) {
815 if (osRaku.PropertySet(&options, key, val))
816 return 0;
817 return -1;
818 }
819
820 /*
821 * LexerRaku::WordListSet
822 * -
823 */
824 Sci_Position SCI_METHOD LexerRaku::WordListSet(int n, const char *wl) {
825 WordList *wordListN = nullptr;
826 switch (n) {
827 case 0:
828 wordListN = &keywords;
829 break;
830 case 1:
831 wordListN = &functions;
832 break;
833 case 2:
834 wordListN = &typesBasic;
835 break;
836 case 3:
837 wordListN = &typesComposite;
838 break;
839 case 4:
840 wordListN = &typesDomainSpecific;
841 break;
842 case 5:
843 wordListN = &typesExceptions;
844 break;
845 case 6:
846 wordListN = &adverbs;
847 break;
848 }
849 Sci_Position firstModification = -1;
850 if (wordListN) {
851 if (wordListN->Set(wl)) {
852 firstModification = 0;
853 }
854 }
855 return firstModification;
856 }
857
858 /*
859 * LexerRaku::ProcessRegexTwinCapture
860 * - processes the transition between a regex pair (two sets of delimiters)
861 * - moves to first new delimiter, if a bracket
862 * - returns true when valid delimiter start found (if bracket)
863 */
864 bool LexerRaku::ProcessRegexTwinCapture(StyleContext &sc, const Sci_Position length,
865 int &type, const DelimPair &dp) {
866
867 if (type == RAKUTYPE_REGEX_S || type == RAKUTYPE_REGEX_TR || type == RAKUTYPE_REGEX_Y) {
868 type = -1; // clear type
869
870 // move past chRegQClose if it was the previous char
871 if (dp.isCloser(sc.chPrev))
872 sc.Forward();
873
874 // no processing needed for non-bracket
875 if (dp.isCloser(dp.opener))
876 return true;
877
878 // move to next opening bracket
879 const Sci_Position len = LengthToNextChar(sc, length);
880 if (sc.GetRelativeCharacter(len) == dp.opener) {
881 sc.Forward(len);
882 return true;
883 }
884 }
885 return false;
886 }
887
888 /*
889 * LexerRaku::ProcessStringVars
890 * - processes a string and highlights any valid variables
891 */
892 void LexerRaku::ProcessStringVars(StyleContext &sc, const Sci_Position length, const int varState) {
893 const int state = sc.state;
894 for (Sci_Position pos = 0; pos < length; pos++) {
895 if (sc.state == varState && !IsWordChar(sc.ch)) {
896 sc.SetState(state);
897 } else if (sc.chPrev != '\\'
898 && (sc.ch == '$' || sc.ch == '@')
899 && IsWordStartChar(sc.chNext)) {
900 sc.SetState(varState);
901 }
902 sc.Forward(); // Next character
903 }
904 }
905 /*
906 * LexerRaku::ProcessValidRegQlangStart
907 * - processes a section of the document range from after a Regex / Q delimiter
908 * - returns true on success
909 * - sets: adverbs, chOpen, chClose, chCount
910 * ref: https://docs.raku.org/language/regexes
911 */
912 bool LexerRaku::ProcessValidRegQlangStart(StyleContext &sc, Sci_Position length, const int type,
913 WordList &wordsAdverbs, DelimPair &dp) {
914 Sci_Position startPos = sc.currentPos;
915 Sci_Position startLen = length;
916 const int target_state = sc.state;
917 int state = SCE_RAKU_DEFAULT;
918 std::string str;
919
920 // find our opening delimiter (and occurrences) / save any adverbs
921 dp.opener = 0; // adverbs can be after the first delimiter
922 bool got_all_adverbs = false; // in Regex statements
923 bool got_ident = false; // regex can have an identifier: 'regex R'
924 sc.SetState(state); // set state default to avoid pre-highlights
925 while ((dp.opener == 0 || !got_all_adverbs) && sc.More()) {
926
927 // move to the next non-space character
928 const bool was_space = IsAWhitespace(sc.ch);
929 if (!got_all_adverbs && was_space) {
930 sc.Forward(LengthToNextChar(sc, length));
931 }
932 length = startLen - (sc.currentPos - startPos); // update length remaining
933
934 // parse / eat an identifier (if type == RAKUTYPE_REGEX)
935 if (dp.opener == 0 && !got_ident && type == RAKUTYPE_REGEX && IsAlphabet(sc.ch)) {
936
937 // eat identifier / account for special adverb :sym<name>
938 bool got_sym = false;
939 while (sc.More()) {
940 sc.SetState(SCE_RAKU_IDENTIFIER);
941 while (sc.More() && (IsAlphaNumeric(sc.chNext)
942 || sc.chNext == '_' || sc.chNext == '-')) {
943 sc.Forward();
944 }
945 sc.Forward();
946 if (got_sym && sc.ch == '>') {
947 sc.SetState(SCE_RAKU_OPERATOR); // '>'
948 sc.Forward();
949 break;
950 } else if (type == RAKUTYPE_REGEX && sc.Match(":sym<")) {
951 sc.SetState(SCE_RAKU_ADVERB); // ':sym'
952 sc.Forward(4);
953 sc.SetState(SCE_RAKU_OPERATOR); // '<'
954 sc.Forward();
955 got_sym = true;
956 } else {
957 break;
958 }
959 }
960 sc.SetState(state);
961 got_ident = true;
962 }
963
964 // parse / save an adverb: RAKUTYPE_REGEX only has adverbs after delim
965 // >= RAKUTYPE_QLANG only has adverbs before delim
966 else if (!got_all_adverbs && sc.ch == ':' && (!(dp.opener == 0 && got_ident)
967 && !(dp.opener > 0 && type >= RAKUTYPE_QLANG))) {
968 sc.SetState(SCE_RAKU_ADVERB);
969 while (IsAlphaNumeric(sc.chNext) && sc.More()) {
970 sc.Forward();
971 str += sc.ch;
972 }
973 str += ' ';
974 sc.Forward();
975 sc.SetState(state);
976 }
977
978 // find starting delimiter
979 else if (dp.opener == 0 && (was_space || IsValidRegOrQAdjacent(sc.ch))
980 && IsValidDelimiter(sc.ch)) { // make sure the delimiter is legal (most are)
981 sc.SetState((state = target_state));// start state here...
982 dp.opener = sc.ch; // this is our delimiter, get count
983 if (type < RAKUTYPE_QLANG) // type is Regex
984 dp.count = 1; // has only one delimiter
985 else
986 dp.count = GetRepeatCharCount(sc, dp.opener, length);
987 sc.Forward(dp.count);
988 }
989
990 // we must have all the adverbs by now...
991 else {
992 if (got_all_adverbs)
993 break; // prevent infinite loop: occurs on missing open char
994 got_all_adverbs = true;
995 }
996 }
997
998 // set word list / find a valid closing delimiter (or bomb!)
999 wordsAdverbs.Set(str.c_str());
1000 dp.closer[0] = GetDelimiterCloseChar(dp.opener);
1001 dp.closer[1] = 0; // no other closer char
1002 return dp.closer[0] > 0;
1003 }
1004
1005 /*
1006 * LexerRaku::LengthToNonWordChar
1007 * - returns the length until the next non "word" character: AlphaNum + '_'
1008 * - also sets all the parsed chars in 's'
1009 */
1010 Sci_Position LexerRaku::LengthToNonWordChar(StyleContext &sc, Sci_Position length,
1011 char *s, const int size, Sci_Position offset) {
1012 Sci_Position len = 0;
1013 Sci_Position max_length = size < length ? size : length;
1014 while (len <= max_length) {
1015 const int ch = sc.GetRelativeCharacter(len + offset);
1016 if (!IsWordChar(ch)) {
1017 s[len] = '\0';
1018 break;
1019 }
1020 s[len] = ch;
1021 len++;
1022 }
1023 s[len + 1] = '\0';
1024 return len;
1025 }
1026
1027 /*
1028 * LexerRaku::Lex
1029 * - Main lexer method
1030 */
1031 void SCI_METHOD LexerRaku::Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) {
1032 LexAccessor styler(pAccess);
1033 DelimPair dpEmbeded; // delimiter pair: embedded comments
1034 DelimPair dpString; // delimiter pair: string
1035 DelimPair dpRegQ; // delimiter pair: Regex / Q Lang
1036 std::string hereDelim; // heredoc delimiter (if in heredoc)
1037 int hereState = 0; // heredoc state to use (Q / QQ)
1038 int numState = 0; // number state / type
1039 short cntDecimal = 0; // number decimal count
1040 std::string wordLast; // last word seen
1041 std::string identLast; // last identifier seen
1042 std::string adverbLast; // last (single) adverb seen
1043 WordList lastAdverbs; // last adverbs seen
1044 Sci_Position len; // temp length value
1045 char s[100]; // temp char string
1046 int typeDetect = -1; // temp type detected (for regex and Q lang)
1047 Sci_Position lengthToEnd; // length until the end of range
1048
1049 // Backtrack to safe start position before complex quoted elements
1050
1051 Sci_PositionU newStartPos = startPos;
1052 if (initStyle != SCE_RAKU_DEFAULT) {
1053 // Backtrack to last SCE_RAKU_DEFAULT or 0
1054 while (newStartPos > 0) {
1055 newStartPos--;
1056 if (styler.StyleAt(newStartPos) == SCE_RAKU_DEFAULT)
1057 break;
1058 }
1059 // Backtrack to start of line before SCE_RAKU_HEREDOC_Q?
1060 if (initStyle == SCE_RAKU_HEREDOC_Q || initStyle == SCE_RAKU_HEREDOC_QQ) {
1061 if (newStartPos > 0) {
1062 newStartPos = styler.LineStart(styler.GetLine(newStartPos));
1063 }
1064 }
1065 } else {
1066 const Sci_Position line = styler.GetLine(newStartPos);
1067 if (line > 0) {
1068 // If the previous line is a start of a q or qq heredoc, backtrack to start of line
1069 const Sci_Position startPreviousLine = styler.LineStart(line-1);
1070 if (ContainsQTo(startPreviousLine, newStartPos, styler)) {
1071 newStartPos = startPreviousLine;
1072 }
1073 }
1074 }
1075
1076
1077 // Re-calculate (any) changed startPos, length and initStyle state
1078 if (newStartPos < startPos) {
1079 initStyle = SCE_RAKU_DEFAULT;
1080 length += startPos - newStartPos;
1081 startPos = newStartPos;
1082 }
1083
1084 // init StyleContext
1085 StyleContext sc(startPos, length, initStyle, styler);
1086
1087 // StyleContext Loop
1088 for (; sc.More(); sc.Forward()) {
1089 lengthToEnd = (length - (sc.currentPos - startPos)); // end of range
1090
1091 /* *** Determine if the current state should terminate ************** *
1092 * Everything within the 'switch' statement processes characters up
1093 * until the end of a syntax highlight section / state.
1094 * ****************************************************************** */
1095 switch (sc.state) {
1096 case SCE_RAKU_OPERATOR:
1097 sc.SetState(SCE_RAKU_DEFAULT);
1098 break; // FIXME: better valid operator sequences needed?
1099 case SCE_RAKU_COMMENTLINE:
1100 if (IsANewLine(sc.ch)) {
1101 sc.SetState(SCE_RAKU_DEFAULT);
1102 }
1103 break;
1104 case SCE_RAKU_COMMENTEMBED:
1105 if ((len = LengthToDelimiter(sc, dpEmbeded, lengthToEnd)) >= 0) {
1106 sc.Forward(len); // Move to end delimiter
1107 sc.SetState(SCE_RAKU_DEFAULT);
1108 } else {
1109 sc.Forward(lengthToEnd); // no end delimiter found
1110 }
1111 break;
1112 case SCE_RAKU_POD:
1113 if (sc.atLineStart && sc.Match("=end pod")) {
1114 sc.Forward(8);
1115 sc.SetState(SCE_RAKU_DEFAULT);
1116 }
1117 break;
1118 case SCE_RAKU_STRING:
1119
1120 // Process the string for variables: move to end delimiter
1121 if ((len = LengthToDelimiter(sc, dpString, lengthToEnd)) >= 0) {
1122 if (dpString.interpol) {
1123 ProcessStringVars(sc, len, SCE_RAKU_STRING_VAR);
1124 } else {
1125 sc.Forward(len);
1126 }
1127 sc.SetState(SCE_RAKU_DEFAULT);
1128 } else {
1129 sc.Forward(lengthToEnd); // no end delimiter found
1130 }
1131 break;
1132 case SCE_RAKU_STRING_Q:
1133 case SCE_RAKU_STRING_QQ:
1134 case SCE_RAKU_STRING_Q_LANG:
1135
1136 // No string: previous char was the delimiter
1137 if (dpRegQ.count == 1 && dpRegQ.isCloser(sc.chPrev)) {
1138 sc.SetState(SCE_RAKU_DEFAULT);
1139 }
1140
1141 // Process the string for variables: move to end delimiter
1142 else if ((len = LengthToDelimiter(sc, dpRegQ, lengthToEnd)) >= 0) {
1143
1144 // set (any) heredoc delimiter string
1145 if (lastAdverbs.InList("to")) {
1146 GetRelativeString(sc, -1, len - dpRegQ.count, hereDelim);
1147 hereState = SCE_RAKU_HEREDOC_Q; // default heredoc state
1148 }
1149
1150 // select variable identifiers
1151 if (sc.state == SCE_RAKU_STRING_QQ || lastAdverbs.InList("qq")) {
1152 ProcessStringVars(sc, len, SCE_RAKU_STRING_VAR);
1153 hereState = SCE_RAKU_HEREDOC_QQ; // potential heredoc state
1154 } else {
1155 sc.Forward(len);
1156 }
1157 sc.SetState(SCE_RAKU_DEFAULT);
1158 } else {
1159 sc.Forward(lengthToEnd); // no end delimiter found
1160 }
1161 break;
1162 case SCE_RAKU_HEREDOC_Q:
1163 case SCE_RAKU_HEREDOC_QQ:
1164 if ((len = LengthToEndHeredoc(sc, styler, lengthToEnd, hereDelim.c_str())) >= 0) {
1165 // select variable identifiers
1166 if (sc.state == SCE_RAKU_HEREDOC_QQ) {
1167 ProcessStringVars(sc, len, SCE_RAKU_STRING_VAR);
1168 } else {
1169 sc.Forward(len);
1170 }
1171 sc.SetState(SCE_RAKU_DEFAULT);
1172 } else {
1173 sc.Forward(lengthToEnd); // no end delimiter found
1174 }
1175 hereDelim.clear(); // clear heredoc delimiter
1176 break;
1177 case SCE_RAKU_REGEX:
1178 // account for typeDetect = RAKUTYPE_REGEX_S/TR/Y
1179 while (sc.state == SCE_RAKU_REGEX) {
1180
1181 // No string: previous char was the delimiter
1182 if (dpRegQ.count == 1 && dpRegQ.isCloser(sc.chPrev)) {
1183 if (ProcessRegexTwinCapture(sc, lengthToEnd, typeDetect, dpRegQ))
1184 continue;
1185 sc.SetState(SCE_RAKU_DEFAULT);
1186 break;
1187 }
1188
1189 // Process the string for variables: move to end delimiter
1190 else if ((len = LengthToDelimiter(sc, dpRegQ, lengthToEnd)) >= 0) {
1191 ProcessStringVars(sc, len, SCE_RAKU_REGEX_VAR);
1192 if (ProcessRegexTwinCapture(sc, lengthToEnd, typeDetect, dpRegQ))
1193 continue;
1194 sc.SetState(SCE_RAKU_DEFAULT);
1195 break;
1196 } else {
1197 sc.Forward(lengthToEnd); // no end delimiter found
1198 break;
1199 }
1200 }
1201 break;
1202 case SCE_RAKU_NUMBER:
1203 if (sc.ch == '.') {
1204 if (sc.chNext == '.') { // '..' is an operator
1205 sc.SetState(SCE_RAKU_OPERATOR);
1206 sc.Forward();
1207 if (sc.chNext == '.') // '...' is also an operator
1208 sc.Forward();
1209 break;
1210 } else if (numState > RAKUNUM_FLOAT_EXP
1211 && (cntDecimal < 1 || numState == RAKUNUM_VERSION)) {
1212 cntDecimal++;
1213 sc.Forward();
1214 } else {
1215 sc.SetState(SCE_RAKU_DEFAULT);
1216 break; // too many decimal places
1217 }
1218 }
1219 switch (numState) {
1220 case RAKUNUM_BINARY:
1221 if (!IsNumberChar(sc.ch, 2))
1222 sc.SetState(SCE_RAKU_DEFAULT);
1223 break;
1224 case RAKUNUM_OCTAL:
1225 if (!IsNumberChar(sc.ch, 8))
1226 sc.SetState(SCE_RAKU_DEFAULT);
1227 break;
1228 case RAKUNUM_HEX:
1229 if (!IsNumberChar(sc.ch, 16))
1230 sc.SetState(SCE_RAKU_DEFAULT);
1231 break;
1232 case RAKUNUM_DECIMAL:
1233 case RAKUNUM_VERSION:
1234 if (!IsNumberChar(sc.ch))
1235 sc.SetState(SCE_RAKU_DEFAULT);
1236 }
1237 break;
1238 case SCE_RAKU_WORD:
1239 case SCE_RAKU_FUNCTION:
1240 case SCE_RAKU_TYPEDEF:
1241 case SCE_RAKU_ADVERB:
1242 sc.SetState(SCE_RAKU_DEFAULT);
1243 break;
1244 case SCE_RAKU_MU:
1245 case SCE_RAKU_POSITIONAL:
1246 case SCE_RAKU_ASSOCIATIVE:
1247 case SCE_RAKU_CALLABLE:
1248 case SCE_RAKU_IDENTIFIER:
1249 case SCE_RAKU_GRAMMAR:
1250 case SCE_RAKU_CLASS:
1251 sc.SetState(SCE_RAKU_DEFAULT);
1252 break;
1253 }
1254
1255 /* *** Determine if a new state should be entered ******************* *
1256 * Everything below here identifies the beginning of a state, all or part
1257 * of the characters within this state are processed here, the rest are
1258 * completed above in the terminate state section.
1259 * ****************************************************************** */
1260 if (sc.state == SCE_RAKU_DEFAULT) {
1261
1262 // --- Single line comment
1263 if (sc.ch == '#') {
1264 sc.SetState(SCE_RAKU_COMMENTLINE);
1265 }
1266
1267 // --- POD block
1268 else if (sc.atLineStart && sc.Match("=begin pod")) {
1269 sc.SetState(SCE_RAKU_POD);
1270 sc.Forward(10);
1271 }
1272
1273 // --- String (normal)
1274 else if (sc.chPrev != '\\' && (IsValidQuoteOpener(sc.ch, dpString, RAKUDELIM_QUOTE))) {
1275 sc.SetState(SCE_RAKU_STRING);
1276 }
1277
1278 // --- String (Q Language) ----------------------------------------
1279 // - https://docs.raku.org/language/quoting
1280 // - Q :adverb :adverb //;
1281 // - q,qx,qw,qq,qqx,qqw,qqww :adverb :adverb //;
1282 else if (IsQLangStartAtScPos(sc, typeDetect, lengthToEnd)) {
1283 int state = SCE_RAKU_STRING_Q_LANG;
1284 Sci_Position forward = 1; // single char ident (default)
1285 if (typeDetect > RAKUTYPE_QLANG) {
1286 state = SCE_RAKU_STRING_Q;
1287 if (typeDetect == RAKUTYPE_STR_WQ)
1288 forward = 0; // no char ident
1289 }
1290 if (typeDetect > RAKUTYPE_STR_Q) {
1291 if (typeDetect == RAKUTYPE_STR_QQ)
1292 state = SCE_RAKU_STRING_QQ;
1293 forward++; // two char ident
1294 }
1295 if (typeDetect > RAKUTYPE_STR_QQ)
1296 forward++; // three char ident
1297 if (typeDetect == RAKUTYPE_STR_QQWW)
1298 forward++; // four char ident
1299
1300 // Proceed: check for a valid character after statement
1301 if (IsValidRegOrQAdjacent(sc.GetRelative(forward)) || typeDetect == RAKUTYPE_QLANG) {
1302 sc.SetState(state);
1303 sc.Forward(forward);
1304 lastAdverbs.Clear();
1305
1306 // Process: adverbs / opening delimiter / adverbs after delim
1307 if (ProcessValidRegQlangStart(sc, lengthToEnd, typeDetect,
1308 lastAdverbs, dpRegQ))
1309 sc.SetState(state);
1310 }
1311 }
1312
1313 // --- Regex (rx/s/m/tr/y) ----------------------------------------
1314 // - https://docs.raku.org/language/regexes
1315 else if ((IsRegexStartAtScPos(sc, typeDetect, setOperator) || regexIdent.InList(wordLast.c_str()))) {
1316 if (typeDetect == -1) { // must be a regex identifier word
1317 wordLast.clear();
1318 typeDetect = RAKUTYPE_REGEX;
1319 }
1320 Sci_Position forward = 0; // no ident (RAKUTYPE_REGEX, RAKUTYPE_REGEX_NORM)
1321 if (typeDetect > 0 && typeDetect != RAKUTYPE_REGEX)
1322 forward++; // single char ident
1323 if (typeDetect > RAKUTYPE_REGEX)
1324 forward++; // two char ident
1325
1326 // Proceed: check for a valid character after statement
1327 if (IsValidRegOrQAdjacent(sc.GetRelative(forward)) || typeDetect == RAKUTYPE_REGEX_NORM) {
1328 sc.SetState(SCE_RAKU_REGEX);
1329 sc.Forward(forward);
1330 lastAdverbs.Clear();
1331
1332 // Process: adverbs / opening delimiter / adverbs after delim
1333 if (ProcessValidRegQlangStart(sc, lengthToEnd, typeDetect,
1334 lastAdverbs, dpRegQ))
1335 sc.SetState(SCE_RAKU_REGEX);
1336 }
1337 }
1338
1339 // --- Numbers ----------------------------------------------------
1340 else if (IsValidIdentPrecede(sc.chPrev) && (IsNumberChar(sc.ch)
1341 || (sc.ch == 'v' && IsNumberChar(sc.chNext) && wordLast == "use"))) {
1342 numState = RAKUNUM_DECIMAL; // default: decimal (base 10)
1343 cntDecimal = 0;
1344 sc.SetState(SCE_RAKU_NUMBER);
1345 if (sc.ch == 'v') // forward past 'v'
1346 sc.Forward();
1347 if (wordLast == "use") { // package version number
1348 numState = RAKUNUM_VERSION;
1349 } else if (sc.ch == '0') { // other type of number
1350 switch (sc.chNext) {
1351 case 'b': // binary (base 2)
1352 numState = RAKUNUM_BINARY;
1353 break;
1354 case 'o': // octal (base 8)
1355 numState = RAKUNUM_OCTAL;
1356 break;
1357 case 'x': // hexadecimal (base 16)
1358 numState = RAKUNUM_HEX;
1359 }
1360 if (numState != RAKUNUM_DECIMAL)
1361 sc.Forward(); // forward to number type char
1362 }
1363 }
1364
1365 // --- Keywords / functions / types / barewords -------------------
1366 else if ((sc.currentPos == 0 || sc.atLineStart || IsValidIdentPrecede(sc.chPrev))
1367 && IsWordStartChar(sc.ch)) {
1368 len = LengthToNonWordChar(sc, lengthToEnd, s, sizeof(s));
1369 if (keywords.InList(s)) {
1370 sc.SetState(SCE_RAKU_WORD); // Keywords
1371 } else if(functions.InList(s)) {
1372 sc.SetState(SCE_RAKU_FUNCTION); // Functions
1373 } else if(typesBasic.InList(s)) {
1374 sc.SetState(SCE_RAKU_TYPEDEF); // Types (basic)
1375 } else if(typesComposite.InList(s)) {
1376 sc.SetState(SCE_RAKU_TYPEDEF); // Types (composite)
1377 } else if(typesDomainSpecific.InList(s)) {
1378 sc.SetState(SCE_RAKU_TYPEDEF); // Types (domain-specific)
1379 } else if(typesExceptions.InList(s)) {
1380 sc.SetState(SCE_RAKU_TYPEDEF); // Types (exceptions)
1381 } else {
1382 if (wordLast == "class")
1383 sc.SetState(SCE_RAKU_CLASS); // a Class ident
1384 else if (wordLast == "grammar")
1385 sc.SetState(SCE_RAKU_GRAMMAR); // a Grammar ident
1386 else
1387 sc.SetState(SCE_RAKU_IDENTIFIER); // Bareword
1388 identLast = s; // save identifier
1389 }
1390 if (adverbLast == "sym") { // special adverb ":sym"
1391 sc.SetState(SCE_RAKU_IDENTIFIER); // treat as identifier
1392 identLast = s; // save identifier
1393 }
1394 if (sc.state != SCE_RAKU_IDENTIFIER)
1395 wordLast = s; // save word
1396 sc.Forward(len - 1); // ...forward past word
1397 }
1398
1399 // --- Adverbs ----------------------------------------------------
1400 else if (sc.ch == ':' && IsWordStartChar(sc.chNext)) {
1401 len = LengthToNonWordChar(sc, lengthToEnd, s, sizeof(s), 1);
1402 if (adverbs.InList(s)) {
1403 sc.SetState(SCE_RAKU_ADVERB); // Adverbs (begin with ':')
1404 adverbLast = s; // save word
1405 sc.Forward(len); // ...forward past word (less offset: 1)
1406 }
1407 }
1408
1409 // --- Identifiers: $mu / @positional / %associative / &callable --
1410 // see: https://docs.raku.org/language/variables
1411 else if (setSigil.Contains(sc.ch) && (setTwigil.Contains(sc.chNext)
1412 || setSpecialVar.Contains(sc.chNext)
1413 || IsWordStartChar(sc.chNext))) {
1414
1415 // State based on sigil
1416 switch (sc.ch) {
1417 case '$': sc.SetState(SCE_RAKU_MU);
1418 break;
1419 case '@': sc.SetState(SCE_RAKU_POSITIONAL);
1420 break;
1421 case '%': sc.SetState(SCE_RAKU_ASSOCIATIVE);
1422 break;
1423 case '&': sc.SetState(SCE_RAKU_CALLABLE);
1424 }
1425 const int state = sc.state;
1426 sc.Forward();
1427 char ch_delim = 0;
1428 if (setSpecialVar.Contains(sc.ch)
1429 && !setWord.Contains(sc.chNext)) { // Process Special Var
1430 ch_delim = -1;
1431 } else if (setTwigil.Contains(sc.ch)) { // Process Twigil
1432 sc.SetState(SCE_RAKU_OPERATOR);
1433 if (sc.ch == '<' && setWord.Contains(sc.chNext))
1434 ch_delim = '>';
1435 sc.Forward();
1436 sc.SetState(state);
1437 }
1438
1439 // Process (any) identifier
1440 if (ch_delim >= 0) {
1441 sc.Forward(LengthToNonWordChar(sc, lengthToEnd, s, sizeof(s)) - 1);
1442 if (ch_delim > 0 && sc.chNext == ch_delim) {
1443 sc.Forward();
1444 sc.SetState(SCE_RAKU_OPERATOR);
1445 }
1446 identLast = s; // save identifier
1447 }
1448 }
1449
1450 // --- Operators --------------------------------------------------
1451 else if (IsOperatorChar(sc.ch)) {
1452 // FIXME: better valid operator sequences needed?
1453 sc.SetState(SCE_RAKU_OPERATOR);
1454 }
1455
1456 // --- Heredoc: begin ---------------------------------------------
1457 else if (!hereDelim.empty() && sc.atLineEnd) {
1458 if (IsANewLine(sc.ch))
1459 sc.Forward(); // skip a possible CRLF situation
1460 sc.SetState(hereState);
1461 }
1462
1463 // Reset words: on operator semi-colon OR '}' (end of statement)
1464 if (sc.state == SCE_RAKU_OPERATOR && (sc.ch == ';' || sc.ch == '}')) {
1465 wordLast.clear();
1466 identLast.clear();
1467 adverbLast.clear();
1468 }
1469 }
1470
1471 /* *** Determine if an "embedded comment" is to be entered ********** *
1472 * This type of embedded comment section, or multi-line comment comes
1473 * after a normal comment has begun... e.g: #`[ ... ]
1474 * ****************************************************************** */
1475 else if (sc.state == SCE_RAKU_COMMENTLINE && sc.chPrev == '#' && sc.ch == '`') {
1476 if (IsBracketOpenChar(sc.chNext)) {
1477 sc.Forward(); // Condition met for "embedded comment"
1478 dpEmbeded.opener = sc.ch;
1479
1480 // Find the opposite (termination) closing bracket (if any)
1481 dpEmbeded.closer[0] = GetBracketCloseChar(dpEmbeded.opener);
1482 if (dpEmbeded.closer[0] > 0) { // Enter "embedded comment"
1483
1484 // Find multiple opening character occurrence
1485 dpEmbeded.count = GetRepeatCharCount(sc, dpEmbeded.opener, lengthToEnd);
1486 sc.SetState(SCE_RAKU_COMMENTEMBED);
1487 sc.Forward(dpEmbeded.count - 1); // incremented in the next loop
1488 }
1489 }
1490 }
1491 }
1492
1493 // And we're done...
1494 sc.Complete();
1495 }
1496
1497 /*
1498 * LexerRaku::Lex
1499 * - Main fold method
1500 * NOTE: although Raku uses and supports UNICODE characters, we're only looking
1501 * at normal chars here, using 'SafeGetCharAt' - for folding purposes
1502 * that is all we need.
1503 */
1504 #define RAKU_HEADFOLD_SHIFT 4
1505 #define RAKU_HEADFOLD_MASK 0xF0
1506 void SCI_METHOD LexerRaku::Fold(Sci_PositionU startPos, Sci_Position length, int /* initStyle */, IDocument *pAccess) {
1507
1508 // init LexAccessor / return if fold option is off
1509 if (!options.fold) return;
1510 LexAccessor styler(pAccess);
1511
1512 // init char and line positions
1513 const Sci_PositionU endPos = startPos + length;
1514 Sci_Position lineCurrent = styler.GetLine(startPos);
1515
1516 // Backtrack to last SCE_RAKU_DEFAULT line
1517 if (startPos > 0 && lineCurrent > 0) {
1518 while (lineCurrent > 0 && styler.StyleAt(startPos) != SCE_RAKU_DEFAULT) {
1519 lineCurrent--;
1520 startPos = styler.LineStart(lineCurrent);
1521 }
1522 lineCurrent = styler.GetLine(startPos);
1523 }
1524 Sci_PositionU lineStart = startPos;
1525 Sci_PositionU lineStartNext = styler.LineStart(lineCurrent + 1);
1526
1527 // init line folding level
1528 int levelPrev = SC_FOLDLEVELBASE;
1529 if (lineCurrent > 0)
1530 levelPrev = styler.LevelAt(lineCurrent - 1) >> 16;
1531 int levelCurrent = levelPrev;
1532
1533 // init char and style variables
1534 char chNext = styler[startPos];
1535 int stylePrev = styler.StyleAt(startPos - 1);
1536 int styleNext = styler.StyleAt(startPos);
1537 int styleNextStartLine = styler.StyleAt(lineStartNext);
1538 int visibleChars = 0;
1539 bool wasCommentMulti = false;
1540
1541 // main loop
1542 for (Sci_PositionU i = startPos; i < endPos; i++) {
1543
1544 // next char, style and flags
1545 const char ch = chNext;
1546 chNext = styler.SafeGetCharAt(i + 1);
1547 const int style = styleNext;
1548 styleNext = styler.StyleAt(i + 1);
1549 const bool atEOL = i == (lineStartNext - 1);
1550 const bool atLineStart = i == lineStart;
1551
1552 // --- Comments / Multi-line / POD ------------------------------------
1553 if (options.foldComment) {
1554
1555 // Multi-line
1556 if (options.foldCommentMultiline) {
1557 if (style == SCE_RAKU_COMMENTLINE && atLineStart && ch == '#' && chNext == '`'
1558 && styleNextStartLine == SCE_RAKU_COMMENTEMBED) {
1559 levelCurrent++;
1560 wasCommentMulti = true; // don't confuse line comments
1561 } else if (style == SCE_RAKU_COMMENTEMBED && atLineStart
1562 && styleNextStartLine != SCE_RAKU_COMMENTEMBED) {
1563 levelCurrent--;
1564 }
1565 }
1566
1567 // Line comments
1568 if (!wasCommentMulti && atEOL && stylePrev == SCE_RAKU_COMMENTLINE
1569 && IsCommentLine(lineCurrent, styler)) {
1570 if (!IsCommentLine(lineCurrent - 1, styler)
1571 && IsCommentLine(lineCurrent + 1, styler))
1572 levelCurrent++;
1573 else if (IsCommentLine(lineCurrent - 1, styler)
1574 && !IsCommentLine(lineCurrent + 1, styler))
1575 levelCurrent--;
1576 }
1577
1578 // POD
1579 if (options.foldCommentPOD && atLineStart && style == SCE_RAKU_POD) {
1580 if (styler.Match(i, "=begin"))
1581 levelCurrent++;
1582 else if (styler.Match(i, "=end"))
1583 levelCurrent--;
1584 }
1585 }
1586
1587 // --- Code block -----------------------------------------------------
1588 if (style == SCE_RAKU_OPERATOR) {
1589 if (ch == '{') {
1590 if (levelCurrent < levelPrev) levelPrev--;
1591 levelCurrent++;
1592 } else if (ch == '}') {
1593 levelCurrent--;
1594 }
1595 }
1596
1597 // --- at end of line / range / apply fold ----------------------------
1598 if (atEOL) {
1599 int level = levelPrev;
1600
1601 // set level flags
1602 level |= levelCurrent << 16;
1603 if (visibleChars == 0 && options.foldCompact)
1604 level |= SC_FOLDLEVELWHITEFLAG;
1605 if ((levelCurrent > levelPrev) && (visibleChars > 0))
1606 level |= SC_FOLDLEVELHEADERFLAG;
1607 if (level != styler.LevelAt(lineCurrent)) {
1608 styler.SetLevel(lineCurrent, level);
1609 }
1610 lineCurrent++;
1611 lineStart = lineStartNext;
1612 lineStartNext = styler.LineStart(lineCurrent + 1);
1613 styleNextStartLine = styler.StyleAt(lineStartNext);
1614 levelPrev = levelCurrent;
1615 visibleChars = 0;
1616 wasCommentMulti = false;
1617 }
1618
1619 // increment visibleChars / set previous char
1620 if (!isspacechar(ch))
1621 visibleChars++;
1622 stylePrev = style;
1623 }
1624
1625 // Done: set real level of the next line
1626 int flagsNext = styler.LevelAt(lineCurrent) & ~SC_FOLDLEVELNUMBERMASK;
1627 styler.SetLevel(lineCurrent, levelPrev | flagsNext);
1628 }
1629
1630 /*----------------------------------------------------------------------------*
1631 * --- Scintilla: LexerModule ---
1632 *----------------------------------------------------------------------------*/
1633
1634 extern const LexerModule lmRaku(SCLEX_RAKU, LexerRaku::LexerFactoryRaku, "raku", rakuWordLists);