Debian Codegraph @

scintilla/lexilla/lexers/LexMarkdown.cxx

raw
1 /******************************************************************
2 * LexMarkdown.cxx
3 *
4 * A simple Markdown lexer for scintilla.
5 *
6 * Includes highlighting for some extra features from the
7 * Pandoc implementation; strikeout, using '#.' as a default
8 * ordered list item marker, and delimited code blocks.
9 *
10 * Limitations:
11 *
12 * Standard indented code blocks are not highlighted at all,
13 * as it would conflict with other indentation schemes. Use
14 * delimited code blocks for blanket highlighting of an
15 * entire code block. Embedded HTML is not highlighted either.
16 * Blanket HTML highlighting has issues, because some Markdown
17 * implementations allow Markdown markup inside of the HTML. Also,
18 * there is a following blank line issue that can't be ignored,
19 * explained in the next paragraph. Embedded HTML and code
20 * blocks would be better supported with language specific
21 * highlighting.
22 *
23 * The highlighting aims to accurately reflect correct syntax,
24 * but a few restrictions are relaxed. Delimited code blocks are
25 * highlighted, even if the line following the code block is not blank.
26 * Requiring a blank line after a block, breaks the highlighting
27 * in certain cases, because of the way Scintilla ends up calling
28 * the lexer.
29 *
30 * Written by Jon Strait - [email protected]
31 *
32 * The License.txt file describes the conditions under which this
33 * software may be distributed.
34 *
35 *****************************************************************/
36
37 #include <stdlib.h>
38 #include <string.h>
39 #include <stdio.h>
40 #include <stdarg.h>
41 #include <assert.h>
42
43 #include <string>
44 #include <string_view>
45
46 #include "ILexer.h"
47 #include "Scintilla.h"
48 #include "SciLexer.h"
49
50 #include "WordList.h"
51 #include "LexAccessor.h"
52 #include "Accessor.h"
53 #include "StyleContext.h"
54 #include "CharacterSet.h"
55 #include "LexerModule.h"
56
57 using namespace Lexilla;
58
59 namespace {
60
61 constexpr bool IsNewline(const int ch) {
62 // sc.GetRelative(i) returns '\0' if out of range
63 return (ch == '\n' || ch == '\r' || ch == '\0');
64 }
65
66 }
67
68 // True if can follow ch down to the end with possibly trailing whitespace
69 // Does not set the state SCE_MARKDOWN_LINE_BEGIN as to allow further processing
70 static bool FollowToLineEnd(const int ch, const int state, const Sci_PositionU endPos, StyleContext &sc) {
71 Sci_Position i = 0;
72 while (sc.GetRelative(++i) == ch)
73 ;
74 // Skip over whitespace
75 while (IsASpaceOrTab(sc.GetRelative(i)) && sc.currentPos + i < endPos)
76 ++i;
77 if (IsNewline(sc.GetRelative(i)) || sc.currentPos + i == endPos) {
78 sc.SetState(state);
79 sc.Forward(i);
80 return true;
81 }
82 else return false;
83 }
84
85 // Set the state on text section from current to length characters,
86 // then set the rest until the newline to default, except for any characters matching token
87 static void SetStateAndZoom(const int state, const Sci_Position length, const int token, StyleContext &sc) {
88 sc.SetState(state);
89 sc.Forward(length);
90 sc.SetState(SCE_MARKDOWN_DEFAULT);
91 sc.Forward();
92 bool started = false;
93 while (sc.More() && !IsNewline(sc.ch)) {
94 if (sc.ch == token && !started) {
95 sc.SetState(state);
96 started = true;
97 }
98 else if (sc.ch != token) {
99 sc.SetState(SCE_MARKDOWN_DEFAULT);
100 started = false;
101 }
102 sc.Forward();
103 }
104 sc.SetState(SCE_MARKDOWN_LINE_BEGIN);
105 }
106
107 // Does the previous line have more than spaces and tabs?
108 static bool HasPrevLineContent(StyleContext &sc) {
109 Sci_Position i = 0;
110 // Go back to the previous newline
111 while ((--i + (Sci_Position)sc.currentPos) >= 0 && !IsNewline(sc.GetRelative(i)))
112 ;
113 while ((--i + (Sci_Position)sc.currentPos) >= 0) {
114 const int ch = sc.GetRelative(i);
115 if (ch == '\n')
116 break;
117 if (!((ch == '\r' || IsASpaceOrTab(ch))))
118 return true;
119 }
120 return false;
121 }
122
123 static bool AtTermStart(StyleContext &sc) {
124 return sc.currentPos == 0 || sc.chPrev == 0 || isspacechar(sc.chPrev);
125 }
126
127 static bool IsCompleteStyleRegion(StyleContext &sc, const char *token) {
128 bool found = false;
129 const size_t start = strlen(token);
130 Sci_Position i = static_cast<Sci_Position>(start);
131 while (!IsNewline(sc.GetRelative(i))) {
132 // make sure an empty pair of single-char tokens doesn't match
133 // with a longer token: {*}{*} != {**}
134 if (sc.GetRelative(i) == *token && sc.GetRelative(i - 1) != *token) {
135 found = start > 1U ? sc.GetRelative(i + 1) == token[1] : true;
136 break;
137 }
138 i++;
139 }
140 return AtTermStart(sc) && found;
141 }
142
143 static bool IsValidHrule(const Sci_PositionU endPos, StyleContext &sc) {
144 int count = 1;
145 Sci_Position i = 0;
146 for (;;) {
147 ++i;
148 int c = sc.GetRelative(i);
149 if (c == sc.ch)
150 ++count;
151 // hit a terminating character
152 else if (!IsASpaceOrTab(c) || sc.currentPos + i == endPos) {
153 // Are we a valid HRULE
154 if ((IsNewline(c) || sc.currentPos + i == endPos) &&
155 count >= 3 && !HasPrevLineContent(sc)) {
156 sc.SetState(SCE_MARKDOWN_HRULE);
157 sc.Forward(i);
158 sc.SetState(SCE_MARKDOWN_LINE_BEGIN);
159 return true;
160 }
161 else {
162 sc.SetState(SCE_MARKDOWN_DEFAULT);
163 return false;
164 }
165 }
166 }
167 }
168
169 static void ColorizeMarkdownDoc(Sci_PositionU startPos, Sci_Position length, int initStyle,
170 WordList **, Accessor &styler) {
171 Sci_PositionU endPos = startPos + length;
172 int precharCount = 0;
173 bool isLinkNameDetecting = false;
174 // Don't advance on a new loop iteration and retry at the same position.
175 // Useful in the corner case of having to start at the beginning file position
176 // in the default state.
177 bool freezeCursor = false;
178
179 // property lexer.markdown.header.eolfill
180 // Set to 1 to highlight all ATX header text.
181 bool headerEOLFill = styler.GetPropertyInt("lexer.markdown.header.eolfill", 0) == 1;
182
183 StyleContext sc(startPos, static_cast<Sci_PositionU>(length), initStyle, styler);
184
185 while (sc.More()) {
186 // Skip past escaped characters
187 if (sc.ch == '\\') {
188 sc.Forward();
189 continue;
190 }
191
192 // A blockquotes resets the line semantics
193 if (sc.state == SCE_MARKDOWN_BLOCKQUOTE)
194 sc.SetState(SCE_MARKDOWN_LINE_BEGIN);
195
196 // Conditional state-based actions
197 if (sc.state == SCE_MARKDOWN_CODE2) {
198 if (sc.Match("``")) {
199 const int closingSpan = (sc.GetRelative(2) == '`') ? 3 : 2;
200 sc.Forward(closingSpan);
201 sc.SetState(SCE_MARKDOWN_DEFAULT);
202 }
203 }
204 else if (sc.state == SCE_MARKDOWN_CODE) {
205 if (sc.ch == '`' && sc.chPrev != ' ')
206 sc.ForwardSetState(SCE_MARKDOWN_DEFAULT);
207 }
208 /* De-activated because it gets in the way of other valid indentation
209 * schemes, for example multiple paragraphs inside a list item.
210 // Code block
211 else if (sc.state == SCE_MARKDOWN_CODEBK) {
212 bool d = true;
213 if (IsNewline(sc.ch)) {
214 if (sc.chNext != '\t') {
215 for (int c = 1; c < 5; ++c) {
216 if (sc.GetRelative(c) != ' ')
217 d = false;
218 }
219 }
220 }
221 else if (sc.atLineStart) {
222 if (sc.ch != '\t' ) {
223 for (int i = 0; i < 4; ++i) {
224 if (sc.GetRelative(i) != ' ')
225 d = false;
226 }
227 }
228 }
229 if (!d)
230 sc.SetState(SCE_MARKDOWN_LINE_BEGIN);
231 }
232 */
233 // Strong
234 else if (sc.state == SCE_MARKDOWN_STRONG1) {
235 if ((sc.Match("**") && sc.chPrev != ' ') || IsNewline(sc.GetRelative(2))) {
236 sc.Forward(2);
237 sc.SetState(SCE_MARKDOWN_DEFAULT);
238 }
239 }
240 else if (sc.state == SCE_MARKDOWN_STRONG2) {
241 if ((sc.Match("__") && sc.chPrev != ' ') || IsNewline(sc.GetRelative(2))) {
242 sc.Forward(2);
243 sc.SetState(SCE_MARKDOWN_DEFAULT);
244 }
245 }
246 // Emphasis
247 else if (sc.state == SCE_MARKDOWN_EM1) {
248 if ((sc.ch == '*' && sc.chPrev != ' ') || IsNewline(sc.chNext))
249 sc.ForwardSetState(SCE_MARKDOWN_DEFAULT);
250 }
251 else if (sc.state == SCE_MARKDOWN_EM2) {
252 if ((sc.ch == '_' && sc.chPrev != ' ') || IsNewline(sc.chNext))
253 sc.ForwardSetState(SCE_MARKDOWN_DEFAULT);
254 }
255 else if (sc.state == SCE_MARKDOWN_CODEBK) {
256 if (sc.atLineStart && sc.Match("~~~")) {
257 Sci_Position i = 1;
258 while (!IsNewline(sc.GetRelative(i)) && sc.currentPos + i < endPos)
259 i++;
260 sc.Forward(i);
261 sc.SetState(SCE_MARKDOWN_DEFAULT);
262 }
263 }
264 else if (sc.state == SCE_MARKDOWN_STRIKEOUT) {
265 if ((sc.Match("~~") && sc.chPrev != ' ') || IsNewline(sc.GetRelative(2))) {
266 sc.Forward(2);
267 sc.SetState(SCE_MARKDOWN_DEFAULT);
268 }
269 }
270 else if (sc.state == SCE_MARKDOWN_LINE_BEGIN) {
271 // Header
272 if (sc.Match("######")) {
273 if (headerEOLFill)
274 sc.SetState(SCE_MARKDOWN_HEADER6);
275 else
276 SetStateAndZoom(SCE_MARKDOWN_HEADER6, 6, '#', sc);
277 }
278 else if (sc.Match("#####")) {
279 if (headerEOLFill)
280 sc.SetState(SCE_MARKDOWN_HEADER5);
281 else
282 SetStateAndZoom(SCE_MARKDOWN_HEADER5, 5, '#', sc);
283 }
284 else if (sc.Match("####")) {
285 if (headerEOLFill)
286 sc.SetState(SCE_MARKDOWN_HEADER4);
287 else
288 SetStateAndZoom(SCE_MARKDOWN_HEADER4, 4, '#', sc);
289 }
290 else if (sc.Match("###")) {
291 if (headerEOLFill)
292 sc.SetState(SCE_MARKDOWN_HEADER3);
293 else
294 SetStateAndZoom(SCE_MARKDOWN_HEADER3, 3, '#', sc);
295 }
296 else if (sc.Match("##")) {
297 if (headerEOLFill)
298 sc.SetState(SCE_MARKDOWN_HEADER2);
299 else
300 SetStateAndZoom(SCE_MARKDOWN_HEADER2, 2, '#', sc);
301 }
302 else if (sc.Match("#")) {
303 // Catch the special case of an unordered list
304 if (sc.chNext == '.' && IsASpaceOrTab(sc.GetRelative(2))) {
305 precharCount = 0;
306 sc.SetState(SCE_MARKDOWN_PRECHAR);
307 }
308 else if (headerEOLFill) {
309 sc.SetState(SCE_MARKDOWN_HEADER1);
310 }
311 else
312 SetStateAndZoom(SCE_MARKDOWN_HEADER1, 1, '#', sc);
313 }
314 // Code block
315 else if (sc.Match("~~~")) {
316 if (!HasPrevLineContent(sc))
317 sc.SetState(SCE_MARKDOWN_CODEBK);
318 else
319 sc.SetState(SCE_MARKDOWN_DEFAULT);
320 }
321 else if (sc.ch == '=') {
322 if (HasPrevLineContent(sc) && FollowToLineEnd('=', SCE_MARKDOWN_HEADER1, endPos, sc)) {
323 if (!headerEOLFill)
324 sc.SetState(SCE_MARKDOWN_LINE_BEGIN);
325 }
326 else
327 sc.SetState(SCE_MARKDOWN_DEFAULT);
328 }
329 else if (sc.ch == '-') {
330 if (HasPrevLineContent(sc) && FollowToLineEnd('-', SCE_MARKDOWN_HEADER2, endPos, sc)) {
331 if (!headerEOLFill)
332 sc.SetState(SCE_MARKDOWN_LINE_BEGIN);
333 }
334 else {
335 precharCount = 0;
336 sc.SetState(SCE_MARKDOWN_PRECHAR);
337 }
338 }
339 else if (IsNewline(sc.ch))
340 sc.SetState(SCE_MARKDOWN_LINE_BEGIN);
341 else {
342 precharCount = 0;
343 sc.SetState(SCE_MARKDOWN_PRECHAR);
344 }
345 }
346
347 // The header lasts until the newline
348 else if (sc.state == SCE_MARKDOWN_HEADER1 || sc.state == SCE_MARKDOWN_HEADER2 ||
349 sc.state == SCE_MARKDOWN_HEADER3 || sc.state == SCE_MARKDOWN_HEADER4 ||
350 sc.state == SCE_MARKDOWN_HEADER5 || sc.state == SCE_MARKDOWN_HEADER6) {
351 if (headerEOLFill) {
352 if (sc.atLineStart) {
353 sc.SetState(SCE_MARKDOWN_LINE_BEGIN);
354 freezeCursor = true;
355 }
356 }
357 else if (IsNewline(sc.ch))
358 sc.SetState(SCE_MARKDOWN_LINE_BEGIN);
359 }
360
361 // New state only within the initial whitespace
362 if (sc.state == SCE_MARKDOWN_PRECHAR) {
363 // Blockquote
364 if (sc.ch == '>' && precharCount < 5)
365 sc.SetState(SCE_MARKDOWN_BLOCKQUOTE);
366 /*
367 // Begin of code block
368 else if (!HasPrevLineContent(sc) && (sc.chPrev == '\t' || precharCount >= 4))
369 sc.SetState(SCE_MARKDOWN_CODEBK);
370 */
371 // HRule - Total of three or more hyphens, asterisks, or underscores
372 // on a line by themselves
373 else if ((sc.ch == '-' || sc.ch == '*' || sc.ch == '_') && IsValidHrule(endPos, sc))
374 ;
375 // Unordered list
376 else if ((sc.ch == '-' || sc.ch == '*' || sc.ch == '+') && IsASpaceOrTab(sc.chNext)) {
377 sc.SetState(SCE_MARKDOWN_ULIST_ITEM);
378 sc.ForwardSetState(SCE_MARKDOWN_DEFAULT);
379 }
380 // Ordered list
381 else if (IsADigit(sc.ch)) {
382 int digitCount = 0;
383 while (IsADigit(sc.GetRelative(++digitCount)))
384 ;
385 if (sc.GetRelative(digitCount) == '.' &&
386 IsASpaceOrTab(sc.GetRelative(digitCount + 1))) {
387 sc.SetState(SCE_MARKDOWN_OLIST_ITEM);
388 sc.Forward(digitCount + 1);
389 sc.SetState(SCE_MARKDOWN_DEFAULT);
390 } else {
391 // a textual number at the margin should be plain text
392 sc.SetState(SCE_MARKDOWN_DEFAULT);
393 }
394 }
395 // Alternate Ordered list
396 else if (sc.ch == '#' && sc.chNext == '.' && IsASpaceOrTab(sc.GetRelative(2))) {
397 sc.SetState(SCE_MARKDOWN_OLIST_ITEM);
398 sc.Forward(2);
399 sc.SetState(SCE_MARKDOWN_DEFAULT);
400 }
401 else if (sc.ch != ' ' || precharCount > 2)
402 sc.SetState(SCE_MARKDOWN_DEFAULT);
403 else
404 ++precharCount;
405 }
406
407 // Any link
408 if (sc.state == SCE_MARKDOWN_LINK) {
409 if (sc.Match("](") && sc.GetRelative(-1) != '\\') {
410 sc.Forward(2);
411 isLinkNameDetecting = true;
412 }
413 else if (sc.Match("]:") && sc.GetRelative(-1) != '\\') {
414 sc.Forward(2);
415 sc.SetState(SCE_MARKDOWN_DEFAULT);
416 }
417 else if (!isLinkNameDetecting && sc.ch == ']' && sc.GetRelative(-1) != '\\') {
418 sc.Forward();
419 sc.SetState(SCE_MARKDOWN_DEFAULT);
420 }
421 else if (isLinkNameDetecting && sc.ch == ')' && sc.GetRelative(-1) != '\\') {
422 sc.Forward();
423 sc.SetState(SCE_MARKDOWN_DEFAULT);
424 isLinkNameDetecting = false;
425 }
426 }
427
428 // New state anywhere in doc
429 if (sc.state == SCE_MARKDOWN_DEFAULT) {
430 if (sc.atLineStart && sc.ch == '#') {
431 sc.SetState(SCE_MARKDOWN_LINE_BEGIN);
432 freezeCursor = true;
433 }
434 // Links and Images
435 if (sc.Match("![")) {
436 sc.SetState(SCE_MARKDOWN_LINK);
437 sc.Forward(1);
438 }
439 else if (sc.ch == '[' && sc.GetRelative(-1) != '\\') {
440 sc.SetState(SCE_MARKDOWN_LINK);
441 }
442 // Code - also a special case for alternate inside spacing
443 else if (sc.Match("``") && sc.GetRelative(3) != ' ' && AtTermStart(sc)) {
444 const int openingSpan = (sc.GetRelative(2) == '`') ? 2 : 1;
445 sc.SetState(SCE_MARKDOWN_CODE2);
446 sc.Forward(openingSpan);
447 }
448 else if (sc.ch == '`' && sc.chNext != ' ' && IsCompleteStyleRegion(sc, "`")) {
449 sc.SetState(SCE_MARKDOWN_CODE);
450 }
451 // Strong
452 else if (sc.Match("**") && sc.GetRelative(2) != ' ' && IsCompleteStyleRegion(sc, "**")) {
453 sc.SetState(SCE_MARKDOWN_STRONG1);
454 sc.Forward();
455 }
456 else if (sc.Match("__") && sc.GetRelative(2) != ' ' && IsCompleteStyleRegion(sc, "__")) {
457 sc.SetState(SCE_MARKDOWN_STRONG2);
458 sc.Forward();
459 }
460 // Emphasis
461 else if (sc.ch == '*' && sc.chNext != ' ' && IsCompleteStyleRegion(sc, "*")) {
462 sc.SetState(SCE_MARKDOWN_EM1);
463 }
464 else if (sc.ch == '_' && sc.chNext != ' ' && IsCompleteStyleRegion(sc, "_")) {
465 sc.SetState(SCE_MARKDOWN_EM2);
466 }
467 // Strikeout
468 else if (sc.Match("~~") && !(sc.GetRelative(2) == '~' || sc.GetRelative(2) == ' ') &&
469 IsCompleteStyleRegion(sc, "~~")) {
470 sc.SetState(SCE_MARKDOWN_STRIKEOUT);
471 sc.Forward();
472 }
473 // Beginning of line
474 else if (IsNewline(sc.ch)) {
475 sc.SetState(SCE_MARKDOWN_LINE_BEGIN);
476 }
477 }
478 // Advance if not holding back the cursor for this iteration.
479 if (!freezeCursor)
480 sc.Forward();
481 freezeCursor = false;
482 }
483 sc.Complete();
484 }
485
486 extern const LexerModule lmMarkdown(SCLEX_MARKDOWN, ColorizeMarkdownDoc, "markdown");