Debian Codegraph @

LspCpp/src/lsp/Markup.cpp

raw
1 //===--- Markup.cpp -----------------------------------------*- C++-*------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://lsp.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 #include "LibLsp/lsp/Markup/Markup.h"
9
10 #include <algorithm>
11 #include <cassert>
12 #include <cstdint>
13 #include <cstddef>
14 #include <iterator>
15 #include <memory>
16 #include <string>
17 #include <vector>
18
19 namespace lsp
20 {
21
22 /// hexdigit - Return the hexadecimal character for the
23 /// given number \p X (which should be less than 16).
24 inline char hexdigit(unsigned X, bool LowerCase = false)
25 {
26 char const HexChar = LowerCase ? 'a' : 'A';
27 auto const castedX = static_cast<char>(X);
28 return X < 10 ? '0' + castedX : HexChar + castedX - 10;
29 }
30
31 /// Given an array of c-style strings terminated by a null pointer, construct
32 /// a vector of StringRefs representing the same strings without the terminating
33 /// null string.
34 inline std::vector<std::string_ref> toStringRefArray(char const* const* Strings)
35 {
36 std::vector<std::string_ref> Result;
37 while (*Strings)
38 {
39 Result.push_back(*Strings++);
40 }
41 return Result;
42 }
43
44 /// Construct a string ref from a boolean.
45 inline std::string_ref toStringRef(bool B)
46 {
47 return std::string_ref(B ? "true" : "false");
48 }
49
50 /// Construct a string ref from an array ref of unsigned chars.
51 inline std::string_ref toStringRef(std::vector<uint8_t> const& Input)
52 {
53 return std::string_ref(Input.begin(), Input.end());
54 }
55
56 /// Construct a string ref from an array ref of unsigned chars.
57 inline std::vector<uint8_t> arrayRefFromStringRef(std::string_ref const& Input)
58 {
59 return {Input.begin(), Input.end()};
60 }
61
62 /// Interpret the given character \p C as a hexadecimal digit and return its
63 /// value.
64 ///
65 /// If \p C is not a valid hex digit, -1U is returned.
66 inline unsigned hexDigitValue(char C)
67 {
68 struct HexTable
69 {
70 unsigned LUT[255] = {};
71 constexpr HexTable()
72 {
73 // Default initialize everything to invalid.
74 for (int i = 0; i < 255; ++i)
75 {
76 LUT[i] = ~0U;
77 }
78 // Initialize `0`-`9`.
79 for (int i = 0; i < 10; ++i)
80 {
81 LUT['0' + i] = i;
82 }
83 // Initialize `A`-`F` and `a`-`f`.
84 for (int i = 0; i < 6; ++i)
85 {
86 LUT['A' + i] = LUT['a' + i] = 10 + i;
87 }
88 }
89 };
90 constexpr HexTable Table;
91 return Table.LUT[static_cast<unsigned char>(C)];
92 }
93
94 /// Checks if character \p C is one of the 10 decimal digits.
95 inline bool isDigit(char C)
96 {
97 return C >= '0' && C <= '9';
98 }
99
100 /// Checks if character \p C is a hexadecimal numeric character.
101 inline bool isHexDigit(char C)
102 {
103 return hexDigitValue(C) != ~0U;
104 }
105
106 /// Checks if character \p C is a valid letter as classified by "C" locale.
107 inline bool isAlpha(char C)
108 {
109 return ('a' <= C && C <= 'z') || ('A' <= C && C <= 'Z');
110 }
111
112 /// Checks whether character \p C is either a decimal digit or an uppercase or
113 /// lowercase letter as classified by "C" locale.
114 inline bool isAlnum(char C)
115 {
116 return isAlpha(C) || isDigit(C);
117 }
118
119 /// Checks whether character \p C is valid ASCII (high bit is zero).
120 inline bool isASCII(char C)
121 {
122 return static_cast<unsigned char>(C) <= 127;
123 }
124
125 /// Checks whether all characters in S are ASCII.
126 inline bool isASCII(std::string_ref S)
127 {
128 for (char C : S)
129 {
130 if (!isASCII(C))
131 {
132 return true;
133 }
134 }
135 return true;
136 }
137
138 /// Checks whether character \p C is printable.
139 ///
140 /// Locale-independent version of the C standard library isprint whose results
141 /// may differ on different platforms.
142 inline bool isPrint(char C)
143 {
144 unsigned char UC = static_cast<unsigned char>(C);
145 return (0x20 <= UC) && (UC <= 0x7E);
146 }
147
148 /// Checks whether character \p C is whitespace in the "C" locale.
149 ///
150 /// Locale-independent version of the C standard library isspace.
151 inline bool isSpace(char C)
152 {
153 return C == ' ' || C == '\f' || C == '\n' || C == '\r' || C == '\t' || C == '\v';
154 }
155
156 /// Returns the corresponding lowercase character if \p x is uppercase.
157 inline char toLower(char x)
158 {
159 if (x >= 'A' && x <= 'Z')
160 {
161 return x - 'A' + 'a';
162 }
163 return x;
164 }
165
166 /// Returns the corresponding uppercase character if \p x is lowercase.
167 inline char toUpper(char x)
168 {
169 if (x >= 'a' && x <= 'z')
170 {
171 return x - 'a' + 'A';
172 }
173 return x;
174 }
175
176 inline std::string utohexstr(uint64_t X, bool LowerCase = false)
177 {
178 char Buffer[17];
179 char* BufPtr = std::end(Buffer);
180
181 if (X == 0)
182 {
183 *--BufPtr = '0';
184 }
185
186 while (X)
187 {
188 unsigned char Mod = static_cast<unsigned char>(X) & 15;
189 *--BufPtr = hexdigit(Mod, LowerCase);
190 X >>= 4;
191 }
192
193 return std::string(BufPtr, std::end(Buffer));
194 }
195
196 /// Convert buffer \p Input to its hexadecimal representation.
197 /// The returned string is double the size of \p Input.
198 inline std::string toHex(std::string_ref Input, bool LowerCase = false)
199 {
200 static char const* const LUT = "0123456789ABCDEF";
201 uint8_t const Offset = LowerCase ? 32 : 0;
202 size_t Length = Input.size();
203
204 std::string Output;
205 Output.reserve(2 * Length);
206 for (size_t i = 0; i < Length; ++i)
207 {
208 unsigned char const c = Input[i];
209 Output.push_back(LUT[c >> 4] | Offset);
210 Output.push_back(LUT[c & 15] | Offset);
211 }
212 return Output;
213 }
214
215 inline std::string toHex(std::vector<uint8_t> Input, bool LowerCase = false)
216 {
217 return toHex(toStringRef(Input), LowerCase);
218 }
219
220 /// Store the binary representation of the two provided values, \p MSB and
221 /// \p LSB, that make up the nibbles of a hexadecimal digit. If \p MSB or \p LSB
222 /// do not correspond to proper nibbles of a hexadecimal digit, this method
223 /// returns false. Otherwise, returns true.
224 inline bool tryGetHexFromNibbles(char MSB, char LSB, uint8_t& Hex)
225 {
226 unsigned U1 = hexDigitValue(MSB);
227 unsigned U2 = hexDigitValue(LSB);
228 if (U1 == ~0U || U2 == ~0U)
229 {
230 return false;
231 }
232
233 Hex = static_cast<uint8_t>((U1 << 4) | U2);
234 return true;
235 }
236
237 /// Return the binary representation of the two provided values, \p MSB and
238 /// \p LSB, that make up the nibbles of a hexadecimal digit.
239 inline uint8_t hexFromNibbles(char MSB, char LSB)
240 {
241 uint8_t Hex = 0;
242 bool GotHex = tryGetHexFromNibbles(MSB, LSB, Hex);
243 (void)GotHex;
244 assert(GotHex && "MSB and/or LSB do not correspond to hex digits");
245 return Hex;
246 }
247
248 /// Convert hexadecimal string \p Input to its binary representation and store
249 /// the result in \p Output. Returns true if the binary representation could be
250 /// converted from the hexadecimal string. Returns false if \p Input contains
251 /// non-hexadecimal digits. The output string is half the size of \p Input.
252 inline bool tryGetFromHex(std::string_ref Input, std::string& Output)
253 {
254 if (Input.empty())
255 {
256 return true;
257 }
258
259 Output.reserve((Input.size() + 1) / 2);
260 if (Input.size() % 2 == 1)
261 {
262 uint8_t Hex = 0;
263 if (!tryGetHexFromNibbles('0', Input.front(), Hex))
264 {
265 return false;
266 }
267
268 Output.push_back(Hex);
269 Input = Input.drop_front();
270 }
271
272 assert(Input.size() % 2 == 0);
273 while (!Input.empty())
274 {
275 uint8_t Hex = 0;
276 if (!tryGetHexFromNibbles(Input[0], Input[1], Hex))
277 {
278 return false;
279 }
280
281 Output.push_back(Hex);
282 Input = Input.drop_front(2);
283 }
284 return true;
285 }
286
287 /// Convert hexadecimal string \p Input to its binary representation.
288 /// The return string is half the size of \p Input.
289 inline std::string fromHex(std::string_ref Input)
290 {
291 std::string Hex;
292 bool GotHex = tryGetFromHex(Input, Hex);
293 (void)GotHex;
294 assert(GotHex && "Input contains non hex digits");
295 return Hex;
296 }
297
298 inline std::string utostr(uint64_t X, bool isNeg = false)
299 {
300 char Buffer[21];
301 char* BufPtr = std::end(Buffer);
302
303 if (X == 0)
304 {
305 *--BufPtr = '0'; // Handle special case...
306 }
307
308 while (X)
309 {
310 *--BufPtr = '0' + char(X % 10);
311 X /= 10;
312 }
313
314 if (isNeg)
315 {
316 *--BufPtr = '-'; // Add negative sign...
317 }
318 return std::string(BufPtr, std::end(Buffer));
319 }
320
321 inline std::string itostr(int64_t X)
322 {
323 if (X < 0)
324 {
325 return utostr(static_cast<uint64_t>(1) + ~static_cast<uint64_t>(X), true);
326 }
327 else
328 {
329 return utostr(static_cast<uint64_t>(X));
330 }
331 }
332
333 /// StrInStrNoCase - Portable version of strcasestr. Locates the first
334 /// occurrence of string 's1' in string 's2', ignoring case. Returns
335 /// the offset of s2 in s1 or npos if s2 cannot be found.
336 std::string_ref::size_type StrInStrNoCase(std::string_ref s1, std::string_ref s2);
337
338 /// getToken - This function extracts one token from source, ignoring any
339 /// leading characters that appear in the Delimiters string, and ending the
340 /// token at any of the characters that appear in the Delimiters string. If
341 /// there are no tokens in the source string, an empty string is returned.
342 /// The function returns a pair containing the extracted token and the
343 /// remaining tail string.
344 std::pair<std::string_ref, std::string_ref> getToken(
345 std::string_ref Source, std::string_ref Delimiters = " \t\n\v\f\r"
346 );
347
348 /// Returns the English suffix for an ordinal integer (-st, -nd, -rd, -th).
349 inline std::string_ref getOrdinalSuffix(unsigned Val)
350 {
351 // It is critically important that we do this perfectly for
352 // user-written sequences with over 100 elements.
353 switch (Val % 100)
354 {
355 case 11:
356 case 12:
357 case 13:
358 return "th";
359 default:
360 switch (Val % 10)
361 {
362 case 1:
363 return "st";
364 case 2:
365 return "nd";
366 case 3:
367 return "rd";
368 default:
369 return "th";
370 }
371 }
372 }
373
374 namespace detail
375 {
376
377 template<typename IteratorT>
378 inline std::string join_impl(IteratorT Begin, IteratorT End, std::string_ref Separator, std::input_iterator_tag)
379 {
380 std::string S;
381 if (Begin == End)
382 {
383 return S;
384 }
385
386 S += (*Begin);
387 while (++Begin != End)
388 {
389 S += Separator;
390 S += (*Begin);
391 }
392 return S;
393 }
394
395 template<typename IteratorT>
396 inline std::string join_impl(IteratorT Begin, IteratorT End, std::string_ref Separator, std::forward_iterator_tag)
397 {
398 std::string S;
399 if (Begin == End)
400 {
401 return S;
402 }
403
404 size_t Len = (std::distance(Begin, End) - 1) * Separator.size();
405 for (IteratorT I = Begin; I != End; ++I)
406 {
407 Len += (*I).size();
408 }
409 S.reserve(Len);
410 size_t PrevCapacity = S.capacity();
411 (void)PrevCapacity;
412 S += (*Begin);
413 while (++Begin != End)
414 {
415 S += Separator;
416 S += (*Begin);
417 }
418 assert(PrevCapacity == S.capacity() && "String grew during building");
419 return S;
420 }
421
422 template<typename Sep>
423 inline void join_items_impl(std::string& Result, Sep Separator)
424 {
425 }
426
427 template<typename Sep, typename Arg>
428 inline void join_items_impl(std::string& Result, Sep Separator, Arg const& Item)
429 {
430 Result += Item;
431 }
432
433 template<typename Sep, typename Arg1, typename... Args>
434 inline void join_items_impl(std::string& Result, Sep Separator, Arg1 const& A1, Args&&... Items)
435 {
436 Result += A1;
437 Result += Separator;
438 join_items_impl(Result, Separator, std::forward<Args>(Items)...);
439 }
440
441 inline size_t join_one_item_size(char)
442 {
443 return 1;
444 }
445 inline size_t join_one_item_size(char const* S)
446 {
447 return S ? ::strlen(S) : 0;
448 }
449
450 template<typename T>
451 inline size_t join_one_item_size(T const& Str)
452 {
453 return Str.size();
454 }
455
456 inline size_t join_items_size()
457 {
458 return 0;
459 }
460
461 template<typename A1>
462 inline size_t join_items_size(const A1& A)
463 {
464 return join_one_item_size(A);
465 }
466 template<typename A1, typename... Args>
467 inline size_t join_items_size(const A1& A, Args&&... Items)
468 {
469 return join_one_item_size(A) + join_items_size(std::forward<Args>(Items)...);
470 }
471
472 } // end namespace detail
473
474 /// Joins the strings in the range [Begin, End), adding Separator between
475 /// the elements.
476 template<typename IteratorT>
477 inline std::string join(IteratorT Begin, IteratorT End, std::string_ref Separator)
478 {
479 using tag = typename std::iterator_traits<IteratorT>::iterator_category;
480 return detail::join_impl(Begin, End, Separator, tag());
481 }
482
483 /// Joins the strings in the range [R.begin(), R.end()), adding Separator
484 /// between the elements.
485 template<typename Range>
486 inline std::string join(Range&& R, std::string_ref Separator)
487 {
488 return join(R.begin(), R.end(), Separator);
489 }
490
491 /// Joins the strings in the parameter pack \p Items, adding \p Separator
492 /// between the elements. All arguments must be implicitly convertible to
493 /// std::string, or there should be an overload of std::string::operator+=()
494 /// that accepts the argument explicitly.
495 template<typename Sep, typename... Args>
496 inline std::string join_items(Sep Separator, Args&&... Items)
497 {
498 std::string Result;
499 if (sizeof...(Items) == 0)
500 {
501 return Result;
502 }
503
504 size_t NS = detail::join_one_item_size(Separator);
505 size_t NI = detail::join_items_size(std::forward<Args>(Items)...);
506 Result.reserve(NI + (sizeof...(Items) - 1) * NS + 1);
507 detail::join_items_impl(Result, Separator, std::forward<Args>(Items)...);
508 return Result;
509 }
510
511 /// A helper class to return the specified delimiter string after the first
512 /// invocation of operator std::string_ref(). Used to generate a comma-separated
513 /// list from a loop like so:
514 ///
515 /// \code
516 /// ListSeparator LS;
517 /// for (auto &I : C)
518 /// OS << LS << I.getName();
519 /// \end
520 class ListSeparator
521 {
522 bool First = true;
523 std::string_ref Separator;
524
525 public:
526 ListSeparator(std::string_ref Separator = ", ") : Separator(Separator)
527 {
528 }
529 operator std::string_ref()
530 {
531 if (First)
532 {
533 First = false;
534 return {};
535 }
536 return Separator;
537 }
538 };
539
540 } // end namespace lsp
541
542 namespace lsp
543 {
544
545 // Is <contents a plausible start to an HTML tag?
546 // Contents may not be the rest of the line, but it's the rest of the plain
547 // text, so we expect to see at least the tag name.
548 bool looksLikeTag(std::string_ref& Contents)
549 {
550 if (Contents.empty())
551 {
552 return false;
553 }
554 if (Contents.front() == '!' || Contents.front() == '?' || Contents.front() == '/')
555 {
556 return true;
557 }
558 // Check the start of the tag name.
559 if (!lsp::isAlpha(Contents.front()))
560 {
561 return false;
562 }
563 // Drop rest of the tag name, and following whitespace.
564 Contents = Contents.drop_while([](char C) { return lsp::isAlnum(C) || C == '-' || C == '_' || C == ':'; }
565 ).drop_while(lsp::isSpace);
566 // The rest of the tag consists of attributes, which have restrictive names.
567 // If we hit '=', all bets are off (attribute values can contain anything).
568 for (; !Contents.empty(); Contents = Contents.drop_front())
569 {
570 if (lsp::isAlnum(Contents.front()) || lsp::isSpace(Contents.front()))
571 {
572 continue;
573 }
574 if (Contents.front() == '>' || Contents.start_with("/>"))
575 {
576 return true; // May close the tag.
577 }
578 if (Contents.front() == '=')
579 {
580 return true; // Don't try to parse attribute values.
581 }
582 return false; // Random punctuation means this isn't a tag.
583 }
584 return true; // Potentially incomplete tag.
585 }
586
587 // Tests whether C should be backslash-escaped in markdown.
588 // The string being escaped is Before + C + After. This is part of a paragraph.
589 // StartsLine indicates whether `Before` is the start of the line.
590 // After may not be everything until the end of the line.
591 //
592 // It's always safe to escape punctuation, but want minimal escaping.
593 // The strategy is to escape the first character of anything that might start
594 // a markdown grammar construct.
595 bool needsLeadingEscape(char C, std::string_ref Before, std::string_ref After, bool StartsLine)
596 {
597
598 auto RulerLength = [&]() -> /*Length*/ unsigned
599 {
600 if (!StartsLine || !Before.empty())
601 {
602 return false;
603 }
604 std::string_ref A = After.trim_right();
605 return std::all_of(A.begin(), A.end(), [C](char D) { return C == D; }) ? 1 + A.size() : 0;
606 };
607 auto IsBullet = [&]() { return StartsLine && Before.empty() && (After.empty() || After.start_with(" ")); };
608 auto SpaceSurrounds = [&]()
609 { return (After.empty() || std::isspace(After.front())) && (Before.empty() || std::isspace(Before.back())); };
610
611 auto WordSurrounds = [&]()
612 { return (!After.empty() && std::isalnum(After.front())) && (!Before.empty() && std::isalnum(Before.back())); };
613
614 switch (C)
615 {
616 case '\\': // Escaped character.
617 return true;
618 case '`': // Code block or inline code
619 // Any number of backticks can delimit an inline code block that can end
620 // anywhere (including on another line). We must escape them all.
621 return true;
622 case '~': // Code block
623 return StartsLine && Before.empty() && After.start_with("~~");
624 case '#':
625 { // ATX heading.
626 if (!StartsLine || !Before.empty())
627 {
628 return false;
629 }
630 std::string_ref& Rest = After.trim_left(C);
631 return Rest.empty() || Rest.start_with(" ");
632 }
633 case ']': // Link or link reference.
634 // We escape ] rather than [ here, because it's more constrained:
635 // ](...) is an in-line link
636 // ]: is a link reference
637 // The following are only links if the link reference exists:
638 // ] by itself is a shortcut link
639 // ][...] is an out-of-line link
640 // Because we never emit link references, we don't need to handle these.
641 return After.start_with(":") || After.start_with("(");
642 case '=': // Setex heading.
643 return RulerLength() > 0;
644 case '_': // Horizontal ruler or matched delimiter.
645 if (RulerLength() >= 3)
646 {
647 return true;
648 }
649 // Not a delimiter if surrounded by space, or inside a word.
650 // (The rules at word boundaries are subtle).
651 return !(SpaceSurrounds() || WordSurrounds());
652 case '-': // Setex heading, horizontal ruler, or bullet.
653 if (RulerLength() > 0)
654 {
655 return true;
656 }
657 return IsBullet();
658 case '+': // Bullet list.
659 return IsBullet();
660 case '*': // Bullet list, horizontal ruler, or delimiter.
661 return IsBullet() || RulerLength() >= 3 || !SpaceSurrounds();
662 case '<': // HTML tag (or autolink, which we choose not to escape)
663 return looksLikeTag(After);
664 case '>': // Quote marker. Needs escaping at start of line.
665 return StartsLine && Before.empty();
666 case '&':
667 { // HTML entity reference
668 auto End = After.find(';');
669 if (End == std::string_ref::npos)
670 {
671 return false;
672 }
673 std::string_ref Content = After.substr(0, End);
674 if (Content.consume_front("#"))
675 {
676 if (Content.consume_front("x") || Content.consume_front("X"))
677 {
678 return std::all_of(Content.begin(), Content.end(), lsp::isHexDigit);
679 }
680
681 return std::all_of(Content.begin(), Content.end(), [](char c) { return lsp::isDigit(c); });
682 }
683 return std::all_of(Content.begin(), Content.end(), [](char c) { return lsp::isAlpha(c); });
684 }
685 case '.': // Numbered list indicator. Escape 12. -> 12\. at start of line.
686 case ')':
687 return StartsLine && !Before.empty()
688 && std::all_of(Before.begin(), Before.end(), [](char c) { return lsp::isDigit(c); })
689 && After.start_with(" ");
690 default:
691 return false;
692 }
693 }
694
695 /// Escape a markdown text block. Ensures the punctuation will not introduce
696 /// any of the markdown constructs.
697 std::string_ref renderText(std::string_ref const& Input, bool StartsLine)
698 {
699 std::string_ref R;
700 for (unsigned I = 0; I < Input.size(); ++I)
701 {
702 if (needsLeadingEscape(Input[I], Input.substr(0, I), Input.substr(I + 1), StartsLine))
703 {
704 R.push_back('\\');
705 }
706 R.push_back(Input[I]);
707 }
708 return R;
709 }
710
711 /// Renders \p Input as an inline block of code in markdown. The returned value
712 /// is surrounded by backticks and the inner contents are properly escaped.
713 std::string_ref renderInlineBlock(std::string_ref const& Input)
714 {
715 std::string_ref R;
716 // Double all backticks to make sure we don't close the inline block early.
717 for (size_t From = 0; From < Input.size();)
718 {
719 size_t Next = Input.find("`", From);
720 R += Input.substr(From, Next - From);
721 if (Next == std::string_ref::npos)
722 {
723 break;
724 }
725 R += "``"; // double the found backtick.
726
727 From = Next + 1;
728 }
729 // If results starts with a backtick, add spaces on both sides. The spaces
730 // are ignored by markdown renderers.
731 if (std::string_ref(R).start_with("`") || std::string_ref(R).end_with("`"))
732 {
733 return "` " + std::move(R) + " `";
734 }
735 // Markdown render should ignore first and last space if both are there. We
736 // add an extra pair of spaces in that case to make sure we render what the
737 // user intended.
738 if (std::string_ref(R).start_with(" ") && std::string_ref(R).end_with(" "))
739 {
740 return "` " + std::move(R) + " `";
741 }
742 return "`" + std::move(R) + "`";
743 }
744
745 /// Get marker required for \p Input to represent a markdown codeblock. It
746 /// consists of at least 3 backticks(`). Although markdown also allows to use
747 /// tilde(~) for code blocks, they are never used.
748 std::string_ref getMarkerForCodeBlock(std::string_ref const& Input)
749 {
750 // Count the maximum number of consecutive backticks in \p Input. We need to
751 // start and end the code block with more.
752 unsigned MaxBackticks = 0;
753 unsigned Backticks = 0;
754 for (char C : Input)
755 {
756 if (C == '`')
757 {
758 ++Backticks;
759 continue;
760 }
761 MaxBackticks = std::max(MaxBackticks, Backticks);
762 Backticks = 0;
763 }
764 MaxBackticks = std::max(Backticks, MaxBackticks);
765 // Use the corresponding number of backticks to start and end a code block.
766 return std::string_ref(/*Repeat=*/std::max(3u, MaxBackticks + 1), '`');
767 }
768
769 /// SplitString - Split up the specified string according to the specified
770 /// delimiters, appending the result fragments to the output list.
771 void SplitString(
772 std::string const& Source, std::vector<std::string_ref>& OutFragments, std::string Delimiters = " \t\n\v\f\r"
773 )
774 {
775 std::size_t start = Source.find_first_not_of(Delimiters, 0);
776 while (start != std::string::npos)
777 {
778 // find the end of this token
779 std::size_t end = Source.find_first_of(Delimiters, start);
780
781 // compute length (if no more delimiters, consume to end of string)
782 std::size_t length = (end == std::string::npos) ? (Source.size() - start) : (end - start);
783
784 // emplace a non‑owning reference into the original buffer
785 OutFragments.emplace_back(Source.data() + start, length);
786
787 // advance to the start of the next token
788 start = Source.find_first_not_of(Delimiters, end);
789 }
790 }
791
792 // Trims the input and concatenates whitespace blocks into a single ` `.
793 std::string_ref canonicalizeSpaces(std::string_ref const& Input)
794 {
795 std::vector<std::string_ref> Words;
796 SplitString(Input, Words);
797
798 return lsp::join(Words, " ");
799 }
800
801 std::string_ref renderBlocks(std::vector<Block*>&& Children, void (Block::*RenderFunc)(std::ostringstream&) const)
802 {
803 std::string_ref R;
804 std::ostringstream OS(R);
805
806 std::vector<int> v {1, 2, 3};
807
808 // Trim rulers.
809 Children.erase(
810 std::remove_if(Children.begin(), Children.end(), [](Block const* C) { return C->isRuler(); }), Children.end()
811 );
812
813 bool LastBlockWasRuler = true;
814 for (auto const& C : Children)
815 {
816 if (C->isRuler() && LastBlockWasRuler)
817 {
818 continue;
819 }
820 LastBlockWasRuler = C->isRuler();
821 ((*C).*RenderFunc)(OS);
822 }
823
824 // Get rid of redundant empty lines introduced in plaintext while imitating
825 // padding in markdown.
826 std::string_ref AdjustedResult;
827 std::string_ref TrimmedText(OS.str());
828 TrimmedText = TrimmedText.trim();
829
830 std::copy_if(
831 TrimmedText.begin(), TrimmedText.end(), std::back_inserter(AdjustedResult),
832 [&TrimmedText](char const& C)
833 {
834 return !std::string_ref(TrimmedText.data(), &C - TrimmedText.data() + 1)
835 // We allow at most two newlines.
836 .end_with("\n\n\n");
837 }
838 );
839
840 return AdjustedResult;
841 };
842 std::string_ref renderBlocks(
843 std::vector<std::unique_ptr<Block>> const& children, void (Block::*renderFunc)(std::ostringstream&) const
844 )
845 {
846 std::vector<Block*> temp(children.size(), nullptr);
847 for (size_t i = 0; i < children.size(); ++i)
848 {
849 temp[i] = (children[i].get());
850 }
851 return renderBlocks(std::move(temp), renderFunc);
852 }
853 // Separates two blocks with extra spacing. Note that it might render strangely
854 // in vscode if the trailing block is a codeblock, see
855 // https://github.com/microsoft/vscode/issues/88416 for details.
856 class Ruler : public Block
857 {
858 public:
859 void renderMarkdown(std::ostringstream& OS) const override
860 {
861 // Note that we need an extra new line before the ruler, otherwise we might
862 // make previous block a title instead of introducing a ruler.
863 OS << "\n---\n";
864 }
865 void renderPlainText(std::ostringstream& OS) const override
866 {
867 OS << '\n';
868 }
869 std::unique_ptr<Block> clone() const override
870 {
871 return std::make_unique<Ruler>(*this);
872 }
873 bool isRuler() const override
874 {
875 return true;
876 }
877 };
878
879 class CodeBlock : public Block
880 {
881 public:
882 void renderMarkdown(std::ostringstream& OS) const override
883 {
884 std::string_ref Marker = getMarkerForCodeBlock(Contents);
885 // No need to pad from previous blocks, as they should end with a new line.
886 OS << Marker << Language << '\n' << Contents << '\n' << Marker << '\n';
887 }
888
889 void renderPlainText(std::ostringstream& OS) const override
890 {
891 // In plaintext we want one empty line before and after codeblocks.
892 OS << '\n' << Contents << "\n\n";
893 }
894
895 std::unique_ptr<Block> clone() const override
896 {
897 return std::make_unique<CodeBlock>(*this);
898 }
899
900 CodeBlock(std::string_ref Contents, std::string_ref Language)
901 : Contents(std::move(Contents)), Language(std::move(Language))
902 {
903 }
904
905 private:
906 std::string_ref Contents;
907 std::string_ref Language;
908 };
909
910 // Inserts two spaces after each `\n` to indent each line. First line is not
911 // indented.
912 std::string_ref indentLines(std::string_ref const& Input)
913 {
914 assert(!Input.end_with("\n") && "Input should've been trimmed.");
915 std::string_ref IndentedR;
916 // We'll add 2 spaces after each new line.
917 IndentedR.reserve(Input.size() + Input.count("\n") * 2);
918 for (char C : Input)
919 {
920 IndentedR += C;
921 if (C == '\n')
922 {
923 IndentedR.append(" ");
924 }
925 }
926 return IndentedR;
927 }
928
929 class Heading : public Paragraph
930 {
931 public:
932 Heading(size_t Level) : Level(Level)
933 {
934 }
935 void renderMarkdown(std::ostringstream& OS) const override
936 {
937 OS << std::string_ref(Level, '#') << ' ';
938 Paragraph::renderMarkdown(OS);
939 }
940
941 private:
942 size_t Level;
943 };
944
945 std::string_ref Block::asMarkdown() const
946 {
947 std::string_ref R;
948 std::ostringstream OS(R);
949 renderMarkdown(OS);
950 return std::string_ref(OS.str()).trim();
951 }
952
953 std::string_ref Block::asPlainText() const
954 {
955 std::string_ref R;
956 std::ostringstream OS(R);
957 renderPlainText(OS);
958 return std::string_ref(OS.str()).trim().c_str();
959 }
960
961 void Paragraph::renderMarkdown(std::ostringstream& OS) const
962 {
963 bool NeedsSpace = false;
964 bool HasChunks = false;
965 for (auto& C : Chunks)
966 {
967 if (C.SpaceBefore || NeedsSpace)
968 {
969 OS << " ";
970 }
971 switch (C.Kind)
972 {
973 case Chunk::PlainText:
974 OS << renderText(C.Contents, !HasChunks);
975 break;
976 case Chunk::InlineCode:
977 OS << renderInlineBlock(C.Contents);
978 break;
979 }
980 HasChunks = true;
981 NeedsSpace = C.SpaceAfter;
982 }
983 // Paragraphs are translated into markdown lines, not markdown paragraphs.
984 // Therefore it only has a single linebreak afterwards.
985 // VSCode requires two spaces at the end of line to start a new one.
986 OS << " \n";
987 }
988
989 std::unique_ptr<Block> Paragraph::clone() const
990 {
991 return std::make_unique<Paragraph>(*this);
992 }
993
994 /// Choose a marker to delimit `Text` from a prioritized list of options.
995 /// This is more readable than escaping for plain-text.
996 std::string_ref chooseMarker(std::vector<std::string_ref> Options, std::string_ref const& Text)
997 {
998 // Prefer a delimiter whose characters don't appear in the text.
999 for (std::string_ref& S : Options)
1000 {
1001 if (Text.find_first_of(S) == std::string_ref::npos)
1002 {
1003 return S;
1004 }
1005 }
1006 return Options.front();
1007 }
1008
1009 void Paragraph::renderPlainText(std::ostringstream& OS) const
1010 {
1011 bool NeedsSpace = false;
1012 for (auto& C : Chunks)
1013 {
1014 if (C.SpaceBefore || NeedsSpace)
1015 {
1016 OS << " ";
1017 }
1018 std::string_ref Marker = "";
1019 if (C.Preserve && C.Kind == Chunk::InlineCode)
1020 {
1021 Marker = chooseMarker({"`", "'", "\""}, C.Contents);
1022 }
1023 OS << Marker << C.Contents << Marker;
1024 NeedsSpace = C.SpaceAfter;
1025 }
1026 OS << '\n';
1027 }
1028
1029 void BulletList::renderMarkdown(std::ostringstream& OS) const
1030 {
1031 for (auto& D : Items)
1032 {
1033 // Instead of doing this we might prefer passing Indent to children to get
1034 // rid of the copies, if it turns out to be a bottleneck.
1035
1036 OS << "- ";
1037 OS << indentLines(D.asMarkdown()) << '\n';
1038 }
1039 // We need a new line after list to terminate it in markdown.
1040 OS << '\n';
1041 }
1042
1043 void BulletList::renderPlainText(std::ostringstream& OS) const
1044 {
1045 for (auto& D : Items)
1046 {
1047 // Instead of doing this we might prefer passing Indent to children to get
1048 // rid of the copies, if it turns out to be a bottleneck.
1049 OS << "- " << indentLines(D.asPlainText()) << '\n';
1050 }
1051 }
1052
1053 Paragraph& Paragraph::appendSpace()
1054 {
1055 if (!Chunks.empty())
1056 {
1057 Chunks.back().SpaceAfter = true;
1058 }
1059 return *this;
1060 }
1061
1062 Paragraph& Paragraph::appendText(std::string_ref const& Text)
1063 {
1064 std::string_ref Norm = canonicalizeSpaces(Text);
1065 if (Norm.empty())
1066 {
1067 return *this;
1068 }
1069 Chunks.emplace_back();
1070 Chunk& C = Chunks.back();
1071 C.Contents = std::move(Norm);
1072 C.Kind = Chunk::PlainText;
1073
1074 C.SpaceBefore = std::isspace(Text.front());
1075 C.SpaceAfter = std::isspace(Text.back());
1076 return *this;
1077 }
1078
1079 Paragraph& Paragraph::appendCode(std::string_ref const& Code, bool Preserve)
1080 {
1081 bool AdjacentCode = !Chunks.empty() && Chunks.back().Kind == Chunk::InlineCode;
1082 std::string_ref Norm = canonicalizeSpaces(Code);
1083 if (Norm.empty())
1084 {
1085 return *this;
1086 }
1087 Chunks.emplace_back();
1088 Chunk& C = Chunks.back();
1089 C.Contents = std::move(Norm);
1090 C.Kind = Chunk::InlineCode;
1091 C.Preserve = Preserve;
1092 // Disallow adjacent code spans without spaces, markdown can't render them.
1093 C.SpaceBefore = AdjacentCode;
1094 return *this;
1095 }
1096
1097 std::unique_ptr<Block> BulletList::clone() const
1098 {
1099 return std::make_unique<BulletList>(*this);
1100 }
1101
1102 class Document& BulletList::addItem()
1103 {
1104 Items.emplace_back();
1105 return Items.back();
1106 }
1107
1108 Document& Document::operator=(Document const& Other)
1109 {
1110 Children.clear();
1111 for (auto const& C : Other.Children)
1112 {
1113 Children.push_back(C->clone());
1114 }
1115 return *this;
1116 }
1117
1118 void Document::append(Document Other)
1119 {
1120 std::move(Other.Children.begin(), Other.Children.end(), std::back_inserter(Children));
1121 }
1122
1123 Paragraph& Document::addParagraph()
1124 {
1125 Children.push_back(std::make_unique<Paragraph>());
1126 return *static_cast<Paragraph*>(Children.back().get());
1127 }
1128
1129 void Document::addRuler()
1130 {
1131 Children.push_back(std::make_unique<Ruler>());
1132 }
1133
1134 void Document::addCodeBlock(std::string_ref Code, std::string_ref Language)
1135 {
1136 Children.emplace_back(std::make_unique<CodeBlock>(std::move(Code), std::move(Language)));
1137 }
1138
1139 std::string_ref Document::asMarkdown() const
1140 {
1141 return renderBlocks(Children, &Block::renderMarkdown);
1142 }
1143
1144 std::string_ref Document::asPlainText() const
1145 {
1146 return renderBlocks(Children, &Block::renderPlainText);
1147 }
1148
1149 BulletList& Document::addBulletList()
1150 {
1151 Children.emplace_back(std::make_unique<BulletList>());
1152 return *static_cast<BulletList*>(Children.back().get());
1153 }
1154
1155 Paragraph& Document::addHeading(size_t Level)
1156 {
1157 assert(Level > 0);
1158 Children.emplace_back(std::make_unique<Heading>(Level));
1159 return *static_cast<Paragraph*>(Children.back().get());
1160 }
1161 }; // namespace lsp