ModuleManager.cpp 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505
  1. //===- ModuleManager.cpp - Module Manager ---------------------------------===//
  2. //
  3. // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
  4. // See https://llvm.org/LICENSE.txt for license information.
  5. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  6. //
  7. //===----------------------------------------------------------------------===//
  8. //
  9. // This file defines the ModuleManager class, which manages a set of loaded
  10. // modules for the ASTReader.
  11. //
  12. //===----------------------------------------------------------------------===//
  13. #include "clang/Serialization/ModuleManager.h"
  14. #include "clang/Basic/FileManager.h"
  15. #include "clang/Basic/LLVM.h"
  16. #include "clang/Lex/HeaderSearch.h"
  17. #include "clang/Lex/ModuleMap.h"
  18. #include "clang/Serialization/GlobalModuleIndex.h"
  19. #include "clang/Serialization/InMemoryModuleCache.h"
  20. #include "clang/Serialization/Module.h"
  21. #include "clang/Serialization/PCHContainerOperations.h"
  22. #include "llvm/ADT/STLExtras.h"
  23. #include "llvm/ADT/SetVector.h"
  24. #include "llvm/ADT/SmallPtrSet.h"
  25. #include "llvm/ADT/SmallVector.h"
  26. #include "llvm/ADT/StringRef.h"
  27. #include "llvm/ADT/iterator.h"
  28. #include "llvm/Support/Chrono.h"
  29. #include "llvm/Support/DOTGraphTraits.h"
  30. #include "llvm/Support/ErrorOr.h"
  31. #include "llvm/Support/GraphWriter.h"
  32. #include "llvm/Support/MemoryBuffer.h"
  33. #include "llvm/Support/VirtualFileSystem.h"
  34. #include <algorithm>
  35. #include <cassert>
  36. #include <memory>
  37. #include <string>
  38. #include <system_error>
  39. using namespace clang;
  40. using namespace serialization;
  41. ModuleFile *ModuleManager::lookupByFileName(StringRef Name) const {
  42. auto Entry = FileMgr.getFile(Name, /*OpenFile=*/false,
  43. /*CacheFailure=*/false);
  44. if (Entry)
  45. return lookup(*Entry);
  46. return nullptr;
  47. }
  48. ModuleFile *ModuleManager::lookupByModuleName(StringRef Name) const {
  49. if (const Module *Mod = HeaderSearchInfo.getModuleMap().findModule(Name))
  50. if (const FileEntry *File = Mod->getASTFile())
  51. return lookup(File);
  52. return nullptr;
  53. }
  54. ModuleFile *ModuleManager::lookup(const FileEntry *File) const {
  55. auto Known = Modules.find(File);
  56. if (Known == Modules.end())
  57. return nullptr;
  58. return Known->second;
  59. }
  60. std::unique_ptr<llvm::MemoryBuffer>
  61. ModuleManager::lookupBuffer(StringRef Name) {
  62. auto Entry = FileMgr.getFile(Name, /*OpenFile=*/false,
  63. /*CacheFailure=*/false);
  64. if (!Entry)
  65. return nullptr;
  66. return std::move(InMemoryBuffers[*Entry]);
  67. }
  68. static bool checkSignature(ASTFileSignature Signature,
  69. ASTFileSignature ExpectedSignature,
  70. std::string &ErrorStr) {
  71. if (!ExpectedSignature || Signature == ExpectedSignature)
  72. return false;
  73. ErrorStr =
  74. Signature ? "signature mismatch" : "could not read module signature";
  75. return true;
  76. }
  77. static void updateModuleImports(ModuleFile &MF, ModuleFile *ImportedBy,
  78. SourceLocation ImportLoc) {
  79. if (ImportedBy) {
  80. MF.ImportedBy.insert(ImportedBy);
  81. ImportedBy->Imports.insert(&MF);
  82. } else {
  83. if (!MF.DirectlyImported)
  84. MF.ImportLoc = ImportLoc;
  85. MF.DirectlyImported = true;
  86. }
  87. }
  88. ModuleManager::AddModuleResult
  89. ModuleManager::addModule(StringRef FileName, ModuleKind Type,
  90. SourceLocation ImportLoc, ModuleFile *ImportedBy,
  91. unsigned Generation,
  92. off_t ExpectedSize, time_t ExpectedModTime,
  93. ASTFileSignature ExpectedSignature,
  94. ASTFileSignatureReader ReadSignature,
  95. ModuleFile *&Module,
  96. std::string &ErrorStr) {
  97. Module = nullptr;
  98. // Look for the file entry. This only fails if the expected size or
  99. // modification time differ.
  100. const FileEntry *Entry;
  101. if (Type == MK_ExplicitModule || Type == MK_PrebuiltModule) {
  102. // If we're not expecting to pull this file out of the module cache, it
  103. // might have a different mtime due to being moved across filesystems in
  104. // a distributed build. The size must still match, though. (As must the
  105. // contents, but we can't check that.)
  106. ExpectedModTime = 0;
  107. }
  108. // Note: ExpectedSize and ExpectedModTime will be 0 for MK_ImplicitModule
  109. // when using an ASTFileSignature.
  110. if (lookupModuleFile(FileName, ExpectedSize, ExpectedModTime, Entry)) {
  111. ErrorStr = "module file out of date";
  112. return OutOfDate;
  113. }
  114. if (!Entry && FileName != "-") {
  115. ErrorStr = "module file not found";
  116. return Missing;
  117. }
  118. // Check whether we already loaded this module, before
  119. if (ModuleFile *ModuleEntry = Modules.lookup(Entry)) {
  120. // Check the stored signature.
  121. if (checkSignature(ModuleEntry->Signature, ExpectedSignature, ErrorStr))
  122. return OutOfDate;
  123. Module = ModuleEntry;
  124. updateModuleImports(*ModuleEntry, ImportedBy, ImportLoc);
  125. return AlreadyLoaded;
  126. }
  127. // Allocate a new module.
  128. auto NewModule = std::make_unique<ModuleFile>(Type, Generation);
  129. NewModule->Index = Chain.size();
  130. NewModule->FileName = FileName.str();
  131. NewModule->File = Entry;
  132. NewModule->ImportLoc = ImportLoc;
  133. NewModule->InputFilesValidationTimestamp = 0;
  134. if (NewModule->Kind == MK_ImplicitModule) {
  135. std::string TimestampFilename = NewModule->getTimestampFilename();
  136. llvm::vfs::Status Status;
  137. // A cached stat value would be fine as well.
  138. if (!FileMgr.getNoncachedStatValue(TimestampFilename, Status))
  139. NewModule->InputFilesValidationTimestamp =
  140. llvm::sys::toTimeT(Status.getLastModificationTime());
  141. }
  142. // Load the contents of the module
  143. if (std::unique_ptr<llvm::MemoryBuffer> Buffer = lookupBuffer(FileName)) {
  144. // The buffer was already provided for us.
  145. NewModule->Buffer = &ModuleCache->addBuiltPCM(FileName, std::move(Buffer));
  146. // Since the cached buffer is reused, it is safe to close the file
  147. // descriptor that was opened while stat()ing the PCM in
  148. // lookupModuleFile() above, it won't be needed any longer.
  149. Entry->closeFile();
  150. } else if (llvm::MemoryBuffer *Buffer =
  151. getModuleCache().lookupPCM(FileName)) {
  152. NewModule->Buffer = Buffer;
  153. // As above, the file descriptor is no longer needed.
  154. Entry->closeFile();
  155. } else if (getModuleCache().shouldBuildPCM(FileName)) {
  156. // Report that the module is out of date, since we tried (and failed) to
  157. // import it earlier.
  158. Entry->closeFile();
  159. return OutOfDate;
  160. } else {
  161. // Open the AST file.
  162. llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buf((std::error_code()));
  163. if (FileName == "-") {
  164. Buf = llvm::MemoryBuffer::getSTDIN();
  165. } else {
  166. // Get a buffer of the file and close the file descriptor when done.
  167. Buf = FileMgr.getBufferForFile(NewModule->File, /*isVolatile=*/false);
  168. }
  169. if (!Buf) {
  170. ErrorStr = Buf.getError().message();
  171. return Missing;
  172. }
  173. NewModule->Buffer = &getModuleCache().addPCM(FileName, std::move(*Buf));
  174. }
  175. // Initialize the stream.
  176. NewModule->Data = PCHContainerRdr.ExtractPCH(*NewModule->Buffer);
  177. // Read the signature eagerly now so that we can check it. Avoid calling
  178. // ReadSignature unless there's something to check though.
  179. if (ExpectedSignature && checkSignature(ReadSignature(NewModule->Data),
  180. ExpectedSignature, ErrorStr))
  181. return OutOfDate;
  182. // We're keeping this module. Store it everywhere.
  183. Module = Modules[Entry] = NewModule.get();
  184. updateModuleImports(*NewModule, ImportedBy, ImportLoc);
  185. if (!NewModule->isModule())
  186. PCHChain.push_back(NewModule.get());
  187. if (!ImportedBy)
  188. Roots.push_back(NewModule.get());
  189. Chain.push_back(std::move(NewModule));
  190. return NewlyLoaded;
  191. }
  192. void ModuleManager::removeModules(
  193. ModuleIterator First,
  194. llvm::SmallPtrSetImpl<ModuleFile *> &LoadedSuccessfully,
  195. ModuleMap *modMap) {
  196. auto Last = end();
  197. if (First == Last)
  198. return;
  199. // Explicitly clear VisitOrder since we might not notice it is stale.
  200. VisitOrder.clear();
  201. // Collect the set of module file pointers that we'll be removing.
  202. llvm::SmallPtrSet<ModuleFile *, 4> victimSet(
  203. (llvm::pointer_iterator<ModuleIterator>(First)),
  204. (llvm::pointer_iterator<ModuleIterator>(Last)));
  205. auto IsVictim = [&](ModuleFile *MF) {
  206. return victimSet.count(MF);
  207. };
  208. // Remove any references to the now-destroyed modules.
  209. for (auto I = begin(); I != First; ++I) {
  210. I->Imports.remove_if(IsVictim);
  211. I->ImportedBy.remove_if(IsVictim);
  212. }
  213. Roots.erase(std::remove_if(Roots.begin(), Roots.end(), IsVictim),
  214. Roots.end());
  215. // Remove the modules from the PCH chain.
  216. for (auto I = First; I != Last; ++I) {
  217. if (!I->isModule()) {
  218. PCHChain.erase(llvm::find(PCHChain, &*I), PCHChain.end());
  219. break;
  220. }
  221. }
  222. // Delete the modules and erase them from the various structures.
  223. for (ModuleIterator victim = First; victim != Last; ++victim) {
  224. Modules.erase(victim->File);
  225. if (modMap) {
  226. StringRef ModuleName = victim->ModuleName;
  227. if (Module *mod = modMap->findModule(ModuleName)) {
  228. mod->setASTFile(nullptr);
  229. }
  230. }
  231. }
  232. // Delete the modules.
  233. Chain.erase(Chain.begin() + (First - begin()), Chain.end());
  234. }
  235. void
  236. ModuleManager::addInMemoryBuffer(StringRef FileName,
  237. std::unique_ptr<llvm::MemoryBuffer> Buffer) {
  238. const FileEntry *Entry =
  239. FileMgr.getVirtualFile(FileName, Buffer->getBufferSize(), 0);
  240. InMemoryBuffers[Entry] = std::move(Buffer);
  241. }
  242. ModuleManager::VisitState *ModuleManager::allocateVisitState() {
  243. // Fast path: if we have a cached state, use it.
  244. if (FirstVisitState) {
  245. VisitState *Result = FirstVisitState;
  246. FirstVisitState = FirstVisitState->NextState;
  247. Result->NextState = nullptr;
  248. return Result;
  249. }
  250. // Allocate and return a new state.
  251. return new VisitState(size());
  252. }
  253. void ModuleManager::returnVisitState(VisitState *State) {
  254. assert(State->NextState == nullptr && "Visited state is in list?");
  255. State->NextState = FirstVisitState;
  256. FirstVisitState = State;
  257. }
  258. void ModuleManager::setGlobalIndex(GlobalModuleIndex *Index) {
  259. GlobalIndex = Index;
  260. if (!GlobalIndex) {
  261. ModulesInCommonWithGlobalIndex.clear();
  262. return;
  263. }
  264. // Notify the global module index about all of the modules we've already
  265. // loaded.
  266. for (ModuleFile &M : *this)
  267. if (!GlobalIndex->loadedModuleFile(&M))
  268. ModulesInCommonWithGlobalIndex.push_back(&M);
  269. }
  270. void ModuleManager::moduleFileAccepted(ModuleFile *MF) {
  271. if (!GlobalIndex || GlobalIndex->loadedModuleFile(MF))
  272. return;
  273. ModulesInCommonWithGlobalIndex.push_back(MF);
  274. }
  275. ModuleManager::ModuleManager(FileManager &FileMgr,
  276. InMemoryModuleCache &ModuleCache,
  277. const PCHContainerReader &PCHContainerRdr,
  278. const HeaderSearch &HeaderSearchInfo)
  279. : FileMgr(FileMgr), ModuleCache(&ModuleCache),
  280. PCHContainerRdr(PCHContainerRdr), HeaderSearchInfo(HeaderSearchInfo) {}
  281. ModuleManager::~ModuleManager() { delete FirstVisitState; }
  282. void ModuleManager::visit(llvm::function_ref<bool(ModuleFile &M)> Visitor,
  283. llvm::SmallPtrSetImpl<ModuleFile *> *ModuleFilesHit) {
  284. // If the visitation order vector is the wrong size, recompute the order.
  285. if (VisitOrder.size() != Chain.size()) {
  286. unsigned N = size();
  287. VisitOrder.clear();
  288. VisitOrder.reserve(N);
  289. // Record the number of incoming edges for each module. When we
  290. // encounter a module with no incoming edges, push it into the queue
  291. // to seed the queue.
  292. SmallVector<ModuleFile *, 4> Queue;
  293. Queue.reserve(N);
  294. llvm::SmallVector<unsigned, 4> UnusedIncomingEdges;
  295. UnusedIncomingEdges.resize(size());
  296. for (ModuleFile &M : llvm::reverse(*this)) {
  297. unsigned Size = M.ImportedBy.size();
  298. UnusedIncomingEdges[M.Index] = Size;
  299. if (!Size)
  300. Queue.push_back(&M);
  301. }
  302. // Traverse the graph, making sure to visit a module before visiting any
  303. // of its dependencies.
  304. while (!Queue.empty()) {
  305. ModuleFile *CurrentModule = Queue.pop_back_val();
  306. VisitOrder.push_back(CurrentModule);
  307. // For any module that this module depends on, push it on the
  308. // stack (if it hasn't already been marked as visited).
  309. for (auto M = CurrentModule->Imports.rbegin(),
  310. MEnd = CurrentModule->Imports.rend();
  311. M != MEnd; ++M) {
  312. // Remove our current module as an impediment to visiting the
  313. // module we depend on. If we were the last unvisited module
  314. // that depends on this particular module, push it into the
  315. // queue to be visited.
  316. unsigned &NumUnusedEdges = UnusedIncomingEdges[(*M)->Index];
  317. if (NumUnusedEdges && (--NumUnusedEdges == 0))
  318. Queue.push_back(*M);
  319. }
  320. }
  321. assert(VisitOrder.size() == N && "Visitation order is wrong?");
  322. delete FirstVisitState;
  323. FirstVisitState = nullptr;
  324. }
  325. VisitState *State = allocateVisitState();
  326. unsigned VisitNumber = State->NextVisitNumber++;
  327. // If the caller has provided us with a hit-set that came from the global
  328. // module index, mark every module file in common with the global module
  329. // index that is *not* in that set as 'visited'.
  330. if (ModuleFilesHit && !ModulesInCommonWithGlobalIndex.empty()) {
  331. for (unsigned I = 0, N = ModulesInCommonWithGlobalIndex.size(); I != N; ++I)
  332. {
  333. ModuleFile *M = ModulesInCommonWithGlobalIndex[I];
  334. if (!ModuleFilesHit->count(M))
  335. State->VisitNumber[M->Index] = VisitNumber;
  336. }
  337. }
  338. for (unsigned I = 0, N = VisitOrder.size(); I != N; ++I) {
  339. ModuleFile *CurrentModule = VisitOrder[I];
  340. // Should we skip this module file?
  341. if (State->VisitNumber[CurrentModule->Index] == VisitNumber)
  342. continue;
  343. // Visit the module.
  344. assert(State->VisitNumber[CurrentModule->Index] == VisitNumber - 1);
  345. State->VisitNumber[CurrentModule->Index] = VisitNumber;
  346. if (!Visitor(*CurrentModule))
  347. continue;
  348. // The visitor has requested that cut off visitation of any
  349. // module that the current module depends on. To indicate this
  350. // behavior, we mark all of the reachable modules as having been visited.
  351. ModuleFile *NextModule = CurrentModule;
  352. do {
  353. // For any module that this module depends on, push it on the
  354. // stack (if it hasn't already been marked as visited).
  355. for (llvm::SetVector<ModuleFile *>::iterator
  356. M = NextModule->Imports.begin(),
  357. MEnd = NextModule->Imports.end();
  358. M != MEnd; ++M) {
  359. if (State->VisitNumber[(*M)->Index] != VisitNumber) {
  360. State->Stack.push_back(*M);
  361. State->VisitNumber[(*M)->Index] = VisitNumber;
  362. }
  363. }
  364. if (State->Stack.empty())
  365. break;
  366. // Pop the next module off the stack.
  367. NextModule = State->Stack.pop_back_val();
  368. } while (true);
  369. }
  370. returnVisitState(State);
  371. }
  372. bool ModuleManager::lookupModuleFile(StringRef FileName,
  373. off_t ExpectedSize,
  374. time_t ExpectedModTime,
  375. const FileEntry *&File) {
  376. if (FileName == "-") {
  377. File = nullptr;
  378. return false;
  379. }
  380. // Open the file immediately to ensure there is no race between stat'ing and
  381. // opening the file.
  382. auto FileOrErr = FileMgr.getFile(FileName, /*OpenFile=*/true,
  383. /*CacheFailure=*/false);
  384. if (!FileOrErr) {
  385. File = nullptr;
  386. return false;
  387. }
  388. File = *FileOrErr;
  389. if ((ExpectedSize && ExpectedSize != File->getSize()) ||
  390. (ExpectedModTime && ExpectedModTime != File->getModificationTime()))
  391. // Do not destroy File, as it may be referenced. If we need to rebuild it,
  392. // it will be destroyed by removeModules.
  393. return true;
  394. return false;
  395. }
  396. #ifndef NDEBUG
  397. namespace llvm {
  398. template<>
  399. struct GraphTraits<ModuleManager> {
  400. using NodeRef = ModuleFile *;
  401. using ChildIteratorType = llvm::SetVector<ModuleFile *>::const_iterator;
  402. using nodes_iterator = pointer_iterator<ModuleManager::ModuleConstIterator>;
  403. static ChildIteratorType child_begin(NodeRef Node) {
  404. return Node->Imports.begin();
  405. }
  406. static ChildIteratorType child_end(NodeRef Node) {
  407. return Node->Imports.end();
  408. }
  409. static nodes_iterator nodes_begin(const ModuleManager &Manager) {
  410. return nodes_iterator(Manager.begin());
  411. }
  412. static nodes_iterator nodes_end(const ModuleManager &Manager) {
  413. return nodes_iterator(Manager.end());
  414. }
  415. };
  416. template<>
  417. struct DOTGraphTraits<ModuleManager> : public DefaultDOTGraphTraits {
  418. explicit DOTGraphTraits(bool IsSimple = false)
  419. : DefaultDOTGraphTraits(IsSimple) {}
  420. static bool renderGraphFromBottomUp() { return true; }
  421. std::string getNodeLabel(ModuleFile *M, const ModuleManager&) {
  422. return M->ModuleName;
  423. }
  424. };
  425. } // namespace llvm
  426. void ModuleManager::viewGraph() {
  427. llvm::ViewGraph(*this, "Modules");
  428. }
  429. #endif