Debian Codegraph @

libcextract/SymbolExternalizer.cpp

raw
1 //===- SymbolExternalizer.cpp - Externalize or rename symbols ---*- C++ -*-===//
2 //
3 // This project is licensed under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Externalize or rename symbols.
11 //
12 //===----------------------------------------------------------------------===//
13
14 /* Author: Giuliano Belinassi */
15
16 #include "SymbolExternalizer.hh"
17 #include "PrettyPrint.hh"
18 #include "Error.hh"
19 #include "ClangCompat.hh"
20 #include "LLVMMisc.hh"
21 #include "IntervalTree.hh"
22 #include "Closure.hh"
23
24 #include <unordered_set>
25 #include <iostream>
26
27 #include "clang/Rewrite/Core/Rewriter.h"
28
29 /* Ban symbols that we are sure to cause problems. */
30
31 /* Although SourceManager has "translateFile" method, it seems unreliable
32 because it may translate to the FileID that do not contain any changed
33 buffer. Hence we do our own thing here, which is look at our own
34 FileEntry => FileID that we are sure to have modifications. */
35 #pragma GCC poison translateFile
36
37 /* Tokenize away the function-like macro stuff or expression, we only want
38 the identifier. */
39 #define TOKEN_VECTOR " ().,;+-*/^|&{}[]<>^&|!\r\n\t"
40
41 /* Return the ranges for all identifiers on the ids vector */
42 template <typename T>
43 static std::vector<std::pair<std::string, SourceRange>>
44 Get_Range_Of_Identifier(const SourceRange &range, const T &ids)
45 {
46 std::vector< std::pair < std::string, SourceRange> > ret = {};
47 StringRef string = PrettyPrint::Get_Source_Text(range);
48
49 /* Create temporary buff, strtok modifies it. */
50 unsigned len = string.size();
51 char buf[len + 1];
52 memcpy(buf, string.data(), len);
53 buf[len] = '\0';
54
55 char *tok = strtok(buf, TOKEN_VECTOR);
56 while (tok != nullptr) {
57 if (ids.find(StringRef(tok)) != ids.end()) {
58 /* Found. */
59 ptrdiff_t distance = (ptrdiff_t)(tok - buf);
60 assert(distance >= 0);
61
62 /* Compute the distance from the original SourceRange of the
63 MacroExpansion. */
64 int32_t offset = (int32_t) distance;
65 SourceLocation start = range.getBegin().getLocWithOffset(offset);
66 SourceLocation end = start.getLocWithOffset(strlen(tok)-1);
67
68 /* Add to the list of output. */
69 ret.push_back(std::make_pair(std::string(tok), SourceRange(start, end)));
70 }
71
72 tok = strtok(nullptr, TOKEN_VECTOR);
73 }
74
75 return ret;
76 }
77
78 static std::vector<std::pair<std::string, SourceRange>>
79 Get_Range_Of_Identifier(const SourceRange &range, const StringRef &id)
80 {
81 std::set<StringRef> ids = { id };
82 return Get_Range_Of_Identifier(range, ids);
83 }
84
85 #define EXTERNALIZED_PREFIX "klpe_"
86 #define RENAME_PREFIX "klpp_"
87
88 using namespace clang;
89 using namespace llvm;
90
91 /* IntervalTree. */
92 using namespace Intervals;
93
94 static SourceRange Get_Range_For_Rewriter(const ASTUnit *ast, const SourceRange &range)
95 {
96 const SourceManager &sm = ast->getSourceManager();
97
98 /* Get a more precise source range of declaration. */
99 SourceLocation start = range.getBegin();
100
101 /* Some declarations start with macro expansion, which the Rewriter
102 class simple rejects. Get one which it will accept. */
103 if (!start.isFileID()) {
104 start = sm.getExpansionLoc(start);
105 }
106
107 SourceLocation end = Lexer::getLocForEndOfToken(
108 range.getEnd(),
109 0,
110 sm,
111 ast->getLangOpts());
112
113 SourceRange new_range(start, end);
114 return new_range;
115 }
116
117 class ExternalizerVisitor: public RecursiveASTVisitor<ExternalizerVisitor>
118 {
119 public:
120 /* Constructor. Should match the FunctionUpdate constructor so we can
121 instantiate it in the Visitor. */
122 ExternalizerVisitor(SymbolExternalizer &se)
123 : SE(se)
124 {}
125
126 enum {
127 VISITOR_CONTINUE = true, // Return this for the AST transversal to continue;
128 VISITOR_STOP = false, // Return this for the AST tranversal to stop completely;
129 };
130
131 /* This method will be used by LLVM whenever it finds a symbol declaration,
132 * being a variable or a method/function.
133 */
134 bool VisitDeclaratorDecl(DeclaratorDecl *decl)
135 {
136 /* As we are dealing with variables that should be externalized, we are only
137 * interested in the symbols that are present in the hash. If the symbol
138 * found is not in the hash, just continue to the next symbol.
139 */
140 SymbolUpdateStatus *sym = SE.getSymbolsUpdateStatus(decl->getName());
141 if (sym == nullptr)
142 return VISITOR_CONTINUE;
143
144 ExternalizationType type = sym->ExtType;
145 if (type == ExternalizationType::STRONG) {
146 if (!sym->Done) {
147 /* If we are dealing with a symbol that is a function, and we are
148 * extracting code with IBT set, do not change the function now with a
149 * prefix. This helps to reduce the number of expanded headers, and also
150 * helps to match the code extracted with teh original source code.
151 */
152 std::string sym_name = decl->getName().str();
153 const std::string new_name = SE.Ibt ? sym_name
154 : EXTERNALIZED_PREFIX + sym_name;
155 DeclaratorDecl *new_decl = SE.Create_Externalized_Var(decl, new_name);
156
157 /* If we found the first instance of the function we want to externalize,
158 then proceed to annotate the Decl so we can later decide what to do with
159 it. */
160 sym->NewName = new_name;
161 sym->OldDecl = decl;
162 sym->NewDecl = new_decl;
163 sym->Done = true;
164 sym->Wrap = !SE.Ibt;
165 /* Log entry is added in Late_Externalize(), once we may discard
166 externalizations due to it not being covered by the closure. */
167 }
168 } else if (type == ExternalizationType::WEAK) {
169 /* Now checks if this is a function or a variable delcaration. */
170 sym->OldDecl = decl;
171 if (FunctionDecl *func = dyn_cast<FunctionDecl>(decl)) {
172 /* In the case it is a function we need to remove its declaration that
173 have a body. */
174 if (func->hasBody()) {
175 FunctionDecl *with_body = func->getDefinition();
176 if (with_body != func) {
177 SE.Remove_Text(with_body->getSourceRange(), 1000);
178 }
179 }
180 }
181 } else if (type == ExternalizationType::RENAME) {
182 /* Get SourceRange where the function identifier is. */
183 SourceRange range = Get_Range_For_Rewriter(SE.AST, decl->getSourceRange());
184
185 auto ids = Get_Range_Of_Identifier(range, decl->getName());
186 assert(ids.size() > 0 && "Decl name do not match required identifier?");
187
188 SourceRange id_range = ids[0].second;
189 const std::string new_name = RENAME_PREFIX + decl->getName().str();
190 sym->NewName = new_name;
191 if (!sym->Done) {
192 /* Only register the first decl rename of the same variable. */
193 sym->OldDecl = decl;
194 SE.Log.push_back({.OldName = decl->getName().str(),
195 .NewName = new_name,
196 .Type = type});
197 sym->Done = true;
198 }
199
200 /* In the case there is a `static` modifier in function, try to drop it. */
201 if (FunctionDecl *fdecl = dyn_cast<FunctionDecl>(decl)) {
202 SE.Drop_Static(fdecl);
203 }
204
205 /* Replace text content of old declaration. */
206 SE.Replace_Text(id_range, new_name, 100);
207
208 sym->Wrap = false;
209 }
210
211 return VISITOR_CONTINUE;
212 }
213
214 /* This visitor will be called by LLVM whenever a symbol is being referenced, being it a
215 * variable or a function call. We are only interested in the symbols that are
216 * present in the SymbolsMap. Once found, it updates the code to the new symbol externalized
217 * name of the symbol. */
218 bool VisitDeclRefExpr(DeclRefExpr *expr)
219 {
220 SourceLocation begin = expr->getBeginLoc();
221 SourceLocation end = expr->getEndLoc();
222 SourceRange range(begin, end);
223
224 ValueDecl *decl = expr->getDecl();
225
226 const StringRef &sym_name = decl->getName();
227 SymbolUpdateStatus *sym = SE.getSymbolsUpdateStatus(sym_name);
228
229 /*
230 * Only execute the code in the visitor if we have already externalized the
231 * symbol
232 */
233 if (sym == nullptr)
234 return VISITOR_CONTINUE;
235
236 /* Get the first effective use, which means an DeclRefExpr of a decl that
237 will not be removed by the Closure. */
238 if (sym->FirstUse == nullptr) {
239 ClosureSet &closure = SE.ClosureVisitor.Get_Closure();
240 ASTUnit *ast = SE.AST;
241 SourceManager &sm = ast->getSourceManager();
242 SourceLocation loc = sm.getExpansionLoc(expr->getLocation());
243 Decl *topdecl = Get_Toplevel_Decl_At_Location(ast, loc);
244
245 /* If the declaration is reachable from the functions we want to extract,
246 then we mark it as FirstUse. */
247 if (closure.Is_Decl_Marked(topdecl)) {
248 sym->FirstUse = expr;
249 }
250 }
251
252 /* We must be careful to ensure that the reference we got is actually
253 written cleanly, e.g. it doesn't come from a macro expansion. */
254 if (sym_name == PrettyPrint::Get_Source_Text(range) && sym->Needs_Sym_Rename()) {
255 /* Issue a text modification. */
256 SE.Replace_Text(range, sym->getUseName(), 100);
257 }
258
259 return VISITOR_CONTINUE;
260 }
261
262 private:
263
264 /** A reference to SymbolExternalizer. */
265 SymbolExternalizer &SE;
266 };
267
268 bool SymbolExternalizer::Drop_Static(FunctionDecl *decl)
269 {
270 if (decl->isStatic()) {
271 SourceRange range = Get_Range_For_Rewriter(AST, decl->getSourceRange());
272
273 auto ids = Get_Range_Of_Identifier(range, StringRef("static"));
274 assert(ids.size() > 0 && "static decl without static keyword?");
275
276 SourceRange static_range = ids[0].second;
277 Remove_Text(static_range, 10);
278
279 /* Update the storage class. */
280 decl->setStorageClass(StorageClass::SC_None);
281
282 return true;
283 }
284
285 return false;
286 }
287
288 template <typename DECL>
289 bool SymbolExternalizer::Drop_Static_Add_Extern(DECL *decl)
290 {
291 if (decl->getStorageClass() == StorageClass::SC_Static) {
292 SourceRange range = Get_Range_For_Rewriter(AST, decl->getSourceRange());
293
294 auto ids = Get_Range_Of_Identifier(range, StringRef("static"));
295 assert(ids.size() > 0 && "static decl without static keyword?");
296
297 SourceRange static_range = ids[0].second;
298 Replace_Text(static_range, "extern", 10);
299
300 /* Update the storage class. */
301 decl->setStorageClass(StorageClass::SC_Extern);
302
303 return true;
304 }
305
306 return false;
307 }
308
309 template <typename DECL>
310 bool SymbolExternalizer::Add_Extern(DECL *decl)
311 {
312 StorageClass storage = decl->getStorageClass();
313 if (storage == StorageClass::SC_Static) {
314 Drop_Static_Add_Extern(decl);
315 return true;
316 } else if (storage == StorageClass::SC_None) {
317 Insert_Text(decl->getBeginLoc(), "extern ");
318 decl->setStorageClass(StorageClass::SC_Extern);
319 return true;
320 }
321
322 return false;
323 }
324
325 /* ---- Delta and TextModifications class ------ */
326
327
328 TextModifications::Delta::Delta(const SourceRange &to_change,
329 const std::string &new_text, int prio)
330 : ToChange(to_change),
331 NewText(new_text),
332 Priority(prio)
333 {
334 static int curr_id = 0;
335 ID = curr_id++;
336 }
337
338 void TextModifications::Sort(void)
339 {
340 auto comparator = [](const Delta& a, const Delta &b) {
341 return a.Priority > b.Priority;
342 };
343
344 /* Sort by descending priority (larger comes first). */
345 std::sort(DeltaList.begin(), DeltaList.end(), comparator);
346 }
347
348 void TextModifications::Insert(Delta &delta)
349 {
350 /* Insert into the vector. */
351 DeltaList.push_back(delta);
352 }
353
354 void TextModifications::Solve(void)
355 {
356 /* Interval Tree to compute intersections of changes. We can't have
357 intersections so if we find those we try to solve them. */
358 IntervalTree<SourceLocation, const Delta&> interval_tree;
359 std::vector<Delta> new_arr;
360
361 /* Sort so that the highest priorities comes first. */
362 Sort();
363
364 /* Get how many Text Modifications we desire. */
365 int n = DeltaList.size();
366
367 /* Given one Text Modification, we must check if there is another text
368 modification that intersects with this one. If there is, we must
369 remove the one with smaller priority (the one that is later on the
370 vector because we sorted it).
371
372 Use an Interval Tree to reduce the complexity from O(n(n-1)/2) to
373 O(n log n). */
374 for (int i = 0; i < n; i++) {
375 const Delta &a = DeltaList[i];
376 auto overlap = interval_tree.findOverlappingIntervals(a.ToChange);
377 /* Check if we already have this range in the tree. */
378 if (overlap.size() > 0) {
379 const Delta &b = overlap[0].value;
380
381 if (a.Priority <= b.Priority || a == b) {
382 // Ignore.
383 } else {
384 /* Intersection with the same priority. Issue an error -- we can't have this. */
385 DiagsClass::Emit_Error("Rewriter ranges with same priority intersects");
386 DiagsClass::Emit_Note(" This one: (priority " + std::to_string(a.Priority) + ')',
387 a.ToChange, SM);
388 DiagsClass::Emit_Note("with this: (priority " + std::to_string(b.Priority) + ')',
389 b.ToChange, SM);
390 throw std::runtime_error("SymbolExternalizer can not continue.");
391 }
392 } else {
393 interval_tree.insert(Interval<SourceLocation, const Delta&>(a.ToChange, a));
394 new_arr.push_back(a);
395 }
396 }
397 DeltaList = new_arr;
398 }
399
400 bool TextModifications::Insert_Into_FileEntryMap(const SourceLocation &loc)
401 {
402 /* Register the changed FileID for retrieving the buffer later. */
403 FileID begin_id = SM.getFileID(loc);
404
405 /* Insert into the list of FileIDs. */
406 OptionalFileEntryRef fentry_ref = SM.getFileEntryRefForID(begin_id);
407 const FileEntry *fentry = nullptr;
408
409 if (fentry_ref.has_value()) {
410 fentry = &fentry_ref->getFileEntry();
411 }
412
413 /* There are some cases where the fentry is known to return NULL. Check if
414 those are the cases we already acknownledged. */
415 if (fentry == nullptr) {
416 PresumedLoc ploc = SM.getPresumedLoc(loc);
417 if (ploc.getFilename() == StringRef("<command line>")) {
418 /* Locations comming from the command line can be ignored. */
419 return false;
420 }
421
422 /* Case where the loc doesn't have proper fileID is known to fail. */
423 assert(loc.isFileID() && "Provided SourceLocation does not have valid FileID");
424
425 /* Crash with assertion. */
426 assert(fentry && "FileEntry is NULL on a non-acknowledged case");
427 }
428
429 /* Insert the FileEntry if we don't have one. */
430 if (FileEntryMap.find(fentry) == FileEntryMap.end()) {
431 /* Insert it. */
432 FileEntryMap[fentry] = std::pair(begin_id, fentry_ref->getName());
433 return true;
434 }
435
436 /* Nothing was inserted. */
437 return false;
438 }
439
440 bool TextModifications::Insert_Into_FileEntryMap(const SourceRange &range)
441 {
442 /* Register the changed FileID for retrieving the buffer later. */
443 FileID begin_id = SM.getFileID(range.getBegin());
444 FileID end_id = SM.getFileID(range.getEnd());
445
446 /* Ensure that the fileIDs are equal. */
447 assert(begin_id == end_id);
448
449 return Insert_Into_FileEntryMap(range.getBegin());
450 }
451
452 void TextModifications::Commit(void)
453 {
454 Solve();
455 int n = DeltaList.size();
456
457 for (int i = 0; i < n; i++) {
458 // Commit Change.
459 Delta &a = DeltaList[i];
460
461 /* Try to insert into the FileEntryMap for commiting the change to the buffer
462 later. */
463 Insert_Into_FileEntryMap(a.ToChange);
464
465 /* Get the text we want to change. We only do this to know its length. */
466 StringRef source_text = Lexer::getSourceText(CharSourceRange::getCharRange(
467 a.ToChange),
468 SM, LO);
469 unsigned len = source_text.size();
470
471 /* ReplaceText(SourceRange, StringRef) version is unreliable in llvm-16. */
472 assert(RW.ReplaceText(a.ToChange.getBegin(), len, a.NewText) == false);
473
474 if (DumpingEnabled) {
475 Dump(i, a);
476 }
477 }
478 }
479
480 void TextModifications::Dump(unsigned num, const Delta &a)
481 {
482 std::string output_file = "/tmp/Externalizer.dump." + std::to_string(num) + ".c";
483 FILE *file = fopen(output_file.c_str(), "w");
484
485 llvm::outs() << "Generating " + output_file + '\n';
486
487 std::string note = std::to_string(num) + " Changing " +
488 PrettyPrint::Get_Source_Text_Raw(a.ToChange).str() +
489 " to " + a.NewText;
490 note = "/*\n" + note + "*/\n";
491 fputs(note.c_str(), file);
492
493 /* Iterate into all files we changed. */
494 for (auto it = FileEntryMap.begin(); it != FileEntryMap.end(); ++it) {
495 /*
496 const FileEntry *fentry = it->first;
497 */
498 FileID id = it->second.first;
499
500 /* Our updated file buffer. */
501 const RewriteBuffer *rewritebuf = RW.getRewriteBufferFor(id);
502
503 /* If we have modifications, then dump the buffer. */
504 if (rewritebuf) {
505 /* The RewriteBuffer object contains a sequence of deltas of the original
506 buffer. Clearly a faster way would be apply those deltas into the
507 target buffer directly, but clang doesn't seems to provide such
508 interface, so we expand them into a temporary string and them pass
509 it to the SourceManager. */
510 std::string modified_str = std::string(rewritebuf->begin(), rewritebuf->end());
511 fputs(modified_str.c_str(), file);
512 fputs("\n\n/* --------- */\n\n", file);
513 }
514 }
515
516 fclose(file);
517 }
518
519 bool TextModifications::Is_Before_Or_Equal(const PresumedLoc &a, const PresumedLoc &b)
520 {
521 unsigned a_line = a.getLine();
522 unsigned a_col = a.getColumn();
523 unsigned b_line = b.getLine();
524 unsigned b_col = b.getColumn();
525
526 unsigned m = std::max(a_col, b_col);
527 unsigned psi_a = m*a_line + a_col;
528 unsigned psi_b = m*b_line + b_col;
529
530 return psi_a <= psi_b;
531 }
532
533 bool TextModifications::Intersects(const SourceRange &a, const SourceRange &b)
534 {
535 PresumedLoc a_begin = SM.getPresumedLoc(a.getBegin());
536 PresumedLoc a_end = SM.getPresumedLoc(a.getEnd());
537 PresumedLoc b_begin = SM.getPresumedLoc(b.getBegin());
538 PresumedLoc b_end = SM.getPresumedLoc(b.getEnd());
539
540 assert(a_begin.getFileID() == a_end.getFileID());
541 assert(b_begin.getFileID() == b_end.getFileID());
542
543 if (a_begin.getFileID() != b_begin.getFileID()) {
544 /* Files are distinct, thus we can't easily determine which comes first. */
545 return false;
546 }
547
548 assert(a_begin.getLine() <= a_end.getLine());
549 assert(b_begin.getLine() <= b_end.getLine());
550
551 return Is_Before_Or_Equal(a_begin, b_end) && Is_Before_Or_Equal(b_begin, a_end);
552 }
553
554 bool TextModifications::Intersects(const Delta &a, const Delta &b)
555 {
556 return Intersects(a.ToChange, b.ToChange);
557 }
558
559 /* ---- End of Deltas class -------- */
560
561 void SymbolExternalizer::Replace_Text(const SourceRange &range, StringRef new_name, int prio)
562 {
563 SourceRange rw_range = Get_Range_For_Rewriter(AST, range);
564 TextModifications::Delta delta(rw_range, new_name.str(), prio);
565 TM.Insert(delta);
566 }
567
568 void SymbolExternalizer::Remove_Text(const SourceRange &range, int prio)
569 {
570 Replace_Text(range, "", prio);
571 }
572
573 void SymbolExternalizer::Insert_Text(const SourceLocation &loc, StringRef text)
574 {
575 /* Ensure we got a SourceLocation with FileID. */
576 SourceLocation floc = loc;
577 if (!floc.isFileID()) {
578 floc = AST->getSourceManager().getExpansionLoc(loc);
579 }
580
581 TM.Insert_Into_FileEntryMap(floc);
582 TM.Get_Rewriter().InsertText(floc, text);
583 }
584
585 VarDecl *SymbolExternalizer::Create_Externalized_Var(DeclaratorDecl *decl, const std::string &name)
586 {
587 /* Hack a new Variable Declaration node in which holds the address of our
588 externalized symbol. We use the information of the old function to
589 build it. For example, assume we want to externalize:
590
591 int f(int);
592
593 then we need to emit:
594
595 static int (*f)(int);
596
597 instead of the first function delcaration. Therefore we need:
598 - The new node's type must be a pointer to the original function type.
599 - The source location of the new node should somewhat match the original
600 symbol.
601 - We should append a `static` to it so it doesn't clash with other
602 livepatches. */
603
604 /* Create a new identifier with name `name`. */
605 IdentifierInfo *id = AST->getPreprocessor().getIdentifierInfo(name);
606
607 /* Get clang's ASTContext, needed to allocate the node. */
608 ASTContext &astctx = AST->getASTContext();
609
610 /* Create a type that is a pointer to the externalized object's type. */
611 QualType pointer_to = astctx.getPointerType(decl->getType());
612
613 /* For IBT, create an extern variable with the same type from the original */
614 if (Ibt)
615 pointer_to = decl->getType();
616
617 /* Get context of decl. */
618 DeclContext *decl_ctx;
619
620 /* Create node. */
621 if (FunctionDecl *fdecl = dyn_cast<FunctionDecl>(decl)) {
622 decl_ctx = fdecl->getParent();
623 } else {
624 decl_ctx = decl->getDeclContext();
625 }
626
627 /*
628 * For normal externalization, create a static variable. When dealing with IBT
629 * restrictions, create an extern variable that will be sorted out later by
630 * the code that is using it, like Linux for example.
631 */
632 StorageClass sc = SC_Static;
633 if (Ibt)
634 sc = SC_Extern;
635
636 VarDecl *ret = VarDecl::Create(astctx, decl_ctx,
637 decl->getBeginLoc(),
638 decl->getEndLoc(),
639 id,
640 pointer_to,
641 nullptr,
642 sc
643 );
644
645 if (!Ibt) {
646 ret->addAttr(UsedAttr::Create(astctx));
647 }
648
649 /* In case the original Decl has TLS storage, we must copy it to the new
650 variable as well. */
651 if (VarDecl *vdecl = dyn_cast<VarDecl>(decl)) {
652 ret->setTSCSpec(vdecl->getTSCSpec());
653
654 /* Libpulp support TLS through some ugly hacking. Warn the user about it
655 for now. */
656 if (Is_TLS(vdecl)) {
657 std::string msg = "Externalization of TLS variable " +
658 vdecl->getName().str() +
659 " requires human intervention to call __tls_get_addr";
660
661
662 DiagsClass::Emit_Warn(msg, vdecl->getSourceRange(),
663 AST->getSourceManager());
664 }
665 }
666
667 /* return node. */
668 return ret;
669 }
670
671 bool SymbolExternalizer::Commit_Changes_To_Source(
672 IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> &ofs,
673 IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> &mfs,
674 std::vector<std::string> &includes_to_expand)
675 {
676 SourceManager &sm = AST->getSourceManager();
677 bool modified = false;
678 bool main_file_inserted = false;
679
680 auto new_ofs = IntrusiveRefCntPtr<vfs::OverlayFileSystem>(
681 new vfs::OverlayFileSystem(vfs::getRealFileSystem()));
682
683 auto new_mfs = IntrusiveRefCntPtr<vfs::InMemoryFileSystem>(
684 new vfs::InMemoryFileSystem);
685
686 new_ofs->pushOverlay(new_mfs);
687
688 /* Commit the text changes. */
689 TM.Commit();
690
691 Rewriter &RW = TM.Get_Rewriter();
692 auto FileEntryMap = TM.Get_FileEntry_Map();
693
694 /* Iterate into all files we may have opened, most probably headers that are
695 #include'd. */
696
697 for (auto it = FileEntryMap.begin(); it != FileEntryMap.end(); ++it) {
698 /* Although SourceManager has "translateFile" method, it seems unreliable
699 because it may translate to the FileID that do not contain any changed
700 buffer. Hence we do our own thing here, which is look at our own
701 FileEntry => FileID that we are sure to have modifications. */
702 const FileEntry *fentry = it->first;
703 FileID id = it->second.first;
704 StringRef filename = it->second.second;
705
706 const RewriteBuffer *rewritebuf = RW.getRewriteBufferFor(id);
707
708 /* If we have modifications, then update the buffer. */
709 if (rewritebuf) {
710 /* The RewriteBuffer object contains a sequence of deltas of the original
711 buffer. Clearly a faster way would be apply those deltas into the
712 target buffer directly, but clang doesn't seems to provide such
713 interface, so we expand them into a temporary string and them pass
714 it to the SourceManager. */
715 std::string modified_str = std::string(rewritebuf->begin(), rewritebuf->end());
716
717 if (new_mfs->addFile(filename,
718 0, MemoryBuffer::getMemBufferCopy(modified_str)) == false) {
719 llvm::outs() << "Unable to add " << filename << " into InMemoryFS.\n";
720 }
721
722 /* In case this is not the main file, we need to mark it for expansion. */
723 if (id != sm.getMainFileID()) {
724 IncludeNode *node = IT.Get(fentry);
725
726 /* Check we can mark the node for expansion. Maybe the expansion rule
727 or the user requested that the header can not be expanded. */
728 if (node->Can_Be_Expanded()) {
729 includes_to_expand.push_back(filename.str());
730 } else {
731 /* Issue an error. */
732 std::string msg = "Necessary header " + filename.str() + " cannot be expanded "+
733 "due to either user or ExpansionPolicy request.";
734 DiagsClass::Emit_Error(msg,
735 node->Get_InclusionDirective()->getSourceRange(),
736 AST->getSourceManager());
737 throw std::runtime_error("SymbolExternalizer is unable to continue");
738 return false;
739 }
740 } else {
741 main_file_inserted = true;
742 }
743
744 modified = true;
745 }
746 }
747
748 /* Make sure we inserted the main file. */
749 if (main_file_inserted == false) {
750 FileID id = sm.getMainFileID();
751 OptionalFileEntryRef fentry_ref = sm.getFileEntryRefForID(id);
752 StringRef filename = fentry_ref->getName();
753
754 const std::string &main_file_content = Get_Modifications_To_Main_File();
755 if (new_mfs->addFile(filename,
756 0,
757 MemoryBuffer::getMemBufferCopy(main_file_content)) == false) {
758 llvm::outs() << "Unable to add " << filename << " into InMemoryFS.\n";
759 }
760 }
761
762 ofs = new_ofs;
763 mfs = new_mfs;
764
765 return modified;
766 }
767
768 std::string SymbolExternalizer::Get_Modifications_To_Main_File(void)
769 {
770
771 SourceManager &sm = AST->getSourceManager();
772
773 /* Our updated file buffer for main file. */
774 FileID main_id = sm.getMainFileID();
775 Rewriter &RW = TM.Get_Rewriter();
776 RewriteBuffer &main_buf = RW.getEditBuffer(main_id);
777
778 return std::string(main_buf.begin(), main_buf.end());
779 }
780
781 /** Given a MacroExpansion object, we try to get the location of where the token
782 appears on it.
783
784 TODO: clang may provide a way of doing this with a tokenizer, so maybe this
785 code can become cleaner with it. */
786 std::vector<std::pair<std::string, SourceRange>>
787 SymbolExternalizer::Get_Range_Of_Identifier_In_Macro_Expansion(const MacroExpansion *exp)
788 {
789 return Get_Range_Of_Identifier(exp->getSourceRange(), SymbolsMap);
790 }
791
792 void SymbolExternalizer::Rewrite_Macros(void)
793 {
794 PreprocessingRecord *rec = AST->getPreprocessor().getPreprocessingRecord();
795
796 for (PreprocessedEntity *entity : *rec) {
797 if (MacroDefinitionRecord *def = dyn_cast<MacroDefinitionRecord>(entity)) {
798 MacroInfo *info = MW.Get_Macro_Info(def);
799
800 if (!info)
801 continue;
802
803 for (const Token tok : info->tokens()) {
804 IdentifierInfo *id_info = tok.getIdentifierInfo();
805 if (!id_info)
806 continue;
807
808 MacroInfo *maybe_macro = MW.Get_Macro_Info(id_info, def->getLocation());
809
810 if (!maybe_macro && !MacroWalker::Is_Identifier_Macro_Argument(info, id_info)) {
811 SymbolUpdateStatus *sym = getSymbolsUpdateStatus(id_info->getName());
812 if (sym && sym->Needs_Sym_Rename())
813 Replace_Text(SourceRange(tok.getLocation(), tok.getLastLoc()), sym->getUseName(), 10);
814 }
815 }
816 } else if (MacroExpansion *exp = dyn_cast<MacroExpansion>(entity)) {
817 /* We must look for references to externalized variables in funcion-like
818 macro expansions on the program's toplevel. */
819 auto ranges = Get_Range_Of_Identifier_In_Macro_Expansion(exp);
820
821 for (auto &tok_range : ranges) {
822 // At this point, tok_range will contain a valid symbol
823 SymbolUpdateStatus *sym = getSymbolsUpdateStatus(tok_range.first);
824 if (sym->Needs_Sym_Rename())
825 Replace_Text(tok_range.second, sym->getUseName(), 10);
826 }
827 }
828 }
829 }
830
831 enum ExternalizationType SymbolExternalizer::Get_Symbol_Ext_Type(const std::string &to_externalize)
832 {
833 /* If the symbol is available in the debuginfo and is an EXTERN symbol, we
834 do not need to rewrite it, but rather we need to erase any declaration
835 with body of it. */
836 if (IA.Can_Decide_Visibility()) {
837 if (IA.Is_Externally_Visible(to_externalize))
838 return ExternalizationType::WEAK;
839
840 return ExternalizationType::STRONG;
841 }
842
843 /* Well, we don't have information so we simply strongly externalize
844 everything. */
845 return ExternalizationType::STRONG;
846 }
847
848 SymbolUpdateStatus *SymbolExternalizer::getSymbolsUpdateStatus(const StringRef &sym)
849 {
850 auto ret = SymbolsMap.find(sym);
851 if (ret == SymbolsMap.end())
852 return nullptr;
853
854 return &ret->second;
855 }
856
857 void SymbolExternalizer::Dump_SymbolsMap(void)
858 {
859 SourceManager &sm = AST->getSourceManager();
860
861 for (auto it = SymbolsMap.begin(); it != SymbolsMap.end(); ++it) {
862 SymbolUpdateStatus &sym = it->getValue();
863 sym.Dump(sm);
864 }
865 }
866
867 void SymbolExternalizer::Compute_SymbolsMap_Late_Insert_Locations(std::vector<SymbolUpdateStatus *> &array)
868 {
869 SourceManager &SM = AST->getSourceManager();
870 ASTContext &ctx = AST->getASTContext();
871
872 /* Insert into an array for the sole purpose of sorting in an ascending
873 SourceLocation order. */
874 for (auto it = SymbolsMap.begin(); it != SymbolsMap.end(); ++it) {
875 SymbolUpdateStatus &sym = it->getValue();
876 if (sym.Is_Used()) {
877 array.push_back(&sym);
878 }
879 }
880
881 if (!AllowLateExternalization) {
882 /* If we can't late-externalize, then there is no point in building this
883 datastructure. The LateInsertLocation will turn invalid and it will
884 fallback to the old method. */
885 return;
886 }
887
888 for (auto array_it = array.begin(); array_it != array.end(); ++array_it) {
889 SymbolUpdateStatus *sym = *array_it;
890 DeclRefExpr *first_use = sym->FirstUse;
891
892 /* In case FirstUse is nullptr, then there is nothing to externalize
893 because it will be removed by the ClosurePass. */
894 if (first_use == nullptr)
895 continue;
896
897 SourceLocation loc_1stuse = SM.getExpansionLoc(first_use->getLocation());
898
899 Decl *topdecl = Get_Toplevel_Decl_At_Location(AST, loc_1stuse);
900 assert(topdecl && "No Toplevel decl encapsulate given expr?");
901
902 /* Check if the declaration have comments. In this case we want to insert
903 it before the comment. */
904 sym->LateInsertLocation = Get_Begin_Loc_Of_Decl_Or_Comment(ctx, topdecl);
905 }
906 }
907
908 void SymbolExternalizer::Drop_Function_Body(FunctionDecl *decl)
909 {
910 if (decl->doesThisDeclarationHaveABody()) {
911 Stmt *body = decl->getBody();
912 Replace_Text(body->getSourceRange(), ";", 1000);
913 }
914 }
915
916 void SymbolExternalizer::Drop_Var_Initializer(VarDecl *decl)
917 {
918 if (decl->hasInit() && decl->getInitStyle() == VarDecl::InitializationStyle::CInit) {
919 Expr *init = decl->getInit();
920 SourceLocation init_loc = init->getSourceRange().getBegin();
921 SourceLocation head = init_loc.getLocWithOffset(-1);
922
923 StringRef text = PrettyPrint::Get_Source_Text(init->getSourceRange());
924 if (text.data() == nullptr) {
925 std::string o;
926 llvm::raw_string_ostream outstr(o);
927
928 decl->setInit(nullptr);
929 decl->print(outstr);
930 decl->setInit(init);
931
932 Replace_Text(decl->getSourceRange(), o, 1000);
933 return;
934 }
935
936 /* Search for the '=' initializer token. */
937 while (true) {
938 StringRef text = PrettyPrint::Get_Source_Text({head, init_loc});
939 if (*text.data() == '=') {
940 break;
941 }
942 head = head.getLocWithOffset(-1);
943 }
944
945 Replace_Text({head, init->getSourceRange().getEnd()}, ";", 1000);
946 }
947 }
948
949 void SymbolExternalizer::Handle_IBT_Ext(SymbolUpdateStatus *sym)
950 {
951 /* Get the last usage (more recent decl), and check if it was the
952 * first usage. If yes, drop the declartion and use the NewDecl form
953 * SymbolUpdateStatus.
954 *
955 * For functions, drop the declarations and change the later
956 * prototypes to have the extern storage type.
957 *
958 * For variables, remove all references but the first one, because
959 * this will also be replaced by outstr (NewDecl from
960 * SymbolUpdateStatus).
961 */
962 Decl *ibt_decl = sym->OldDecl->getMostRecentDecl();
963 if (FunctionDecl *func = dyn_cast<FunctionDecl>(ibt_decl)) {
964 while (func) {
965 Drop_Function_Body(func);
966 Drop_Static_Add_Extern(func);
967
968 func = func->getPreviousDecl();
969 }
970 } else if (VarDecl *var = dyn_cast<VarDecl>(ibt_decl)) {
971 while (var) {
972 Add_Extern(var);
973 Drop_Var_Initializer(var);
974
975 var = var->getPreviousDecl();
976 }
977 }
978 }
979
980 void SymbolExternalizer::Late_Externalize(void)
981 {
982 SymbolExternalizer &SE = *this;
983 SourceManager &sm = AST->getSourceManager();
984 ASTContext &astctx = AST->getASTContext();
985 std::vector<SymbolUpdateStatus *> array;
986 Compute_SymbolsMap_Late_Insert_Locations(array);
987
988 for (SymbolUpdateStatus *sym : array) {
989 if (sym->NewDecl == nullptr || sym->ExtType != ExternalizationType::STRONG) {
990 /* Doesn't require strong externalization. */
991 continue;
992 }
993
994 if (Ibt) {
995 /* When IBT is enabled we handle things a little differently. We don't
996 do Late Externalization but rather change the symbols in where they
997 were defined once we need to change them to extern and drop bodies. */
998 Handle_IBT_Ext(sym);
999
1000 /* Remember that we externalized it. */
1001 Log.push_back({.OldName = sym->OldDecl->getName().str(),
1002 .NewName = sym->NewDecl->getName().str(),
1003 .Type = sym->ExtType});
1004 continue;
1005 }
1006
1007 /* Create a string with the new variable type and name. */
1008 std::string o;
1009 llvm::raw_string_ostream outstr(o);
1010
1011 /*
1012 * Get the location of the original decl so we can output the right comment
1013 * location when LateExternalization is active.
1014 */
1015 if (sym->LateInsertLocation.isValid()) {
1016 outstr << Get_Or_Build_CE_Location_Comment(astctx, sym->OldDecl);
1017 }
1018 sym->NewDecl->print(outstr);
1019
1020 std::string sym_name = sym->OldDecl->getName().str();
1021 outstr << ";\n";
1022
1023 /* In case we successfully have a late insertion location, put the new decl
1024 there. */
1025 if (sym->LateInsertLocation.isValid()) {
1026 SE.Insert_Text(sym->LateInsertLocation, outstr.str());
1027
1028 /* In case the symbol is in the main file already, we must delete it. */
1029 DeclaratorDecl *old_decl = Get_With_Body_Or_Itself(sym->OldDecl);
1030
1031 SourceLocation loc = sm.getExpansionLoc(old_decl->getBeginLoc());
1032 if (sm.getFileID(loc) == sm.getMainFileID()) {
1033 SE.Remove_Text(old_decl->getSourceRange(), 1000);
1034 }
1035 } else {
1036 /* Fallback to the old method of rewriting the declaration. */
1037 SE.Replace_Text(sym->OldDecl->getSourceRange(), outstr.str(), 1000);
1038
1039 /* Emit a warning for debuging purposes for now. */
1040 if (AllowLateExternalization) {
1041 std::string msg = "LateLocation of " + sym->OldDecl->getName().str() + " is invalid\n";
1042 DiagsClass::Emit_Warn(msg);
1043 }
1044 }
1045
1046 /* Remember that we externalized it. */
1047 Log.push_back({.OldName = sym->OldDecl->getName().str(),
1048 .NewName = sym->NewDecl->getName().str(),
1049 .Type = sym->ExtType});
1050 }
1051 }
1052
1053 void SymbolExternalizer::Externalize_Symbols(std::vector<std::string> const &to_externalize_array,
1054 std::vector<std::string> &to_rename_array)
1055 {
1056 for (const std::string &to_externalize : to_externalize_array) {
1057 SymbolsMap.insert({to_externalize, SymbolUpdateStatus(Get_Symbol_Ext_Type(to_externalize))});
1058 }
1059
1060 for (std::string &to_externalize : to_rename_array) {
1061 SymbolsMap.insert({to_externalize, SymbolUpdateStatus(ExternalizationType::RENAME)});
1062 to_externalize = RENAME_PREFIX + to_externalize;
1063 }
1064
1065 /* Start traversing the AST to find all references to the symbols that we want
1066 * to externalize or rename. */
1067 ExternalizerVisitor(*this).TraverseDecl(AST->getASTContext().getTranslationUnitDecl());
1068 Late_Externalize();
1069
1070 /* Search for all macros and macro expansions and rewrite them using the new
1071 * names for the externalized variables. */
1072 Rewrite_Macros();
1073 }