VirtualFileSystem.cpp 55 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680
  1. //===- VirtualFileSystem.cpp - Virtual File System Layer --------*- C++ -*-===//
  2. //
  3. // The LLVM Compiler Infrastructure
  4. //
  5. // This file is distributed under the University of Illinois Open Source
  6. // License. See LICENSE.TXT for details.
  7. //
  8. //===----------------------------------------------------------------------===//
  9. // This file implements the VirtualFileSystem interface.
  10. //===----------------------------------------------------------------------===//
  11. #include "clang/Basic/VirtualFileSystem.h"
  12. #include "clang/Basic/FileManager.h"
  13. #include "llvm/ADT/DenseMap.h"
  14. #include "llvm/ADT/STLExtras.h"
  15. #include "llvm/ADT/StringExtras.h"
  16. #include "llvm/ADT/StringSet.h"
  17. #include "llvm/ADT/iterator_range.h"
  18. #include "llvm/Support/Errc.h"
  19. #include "llvm/Support/MemoryBuffer.h"
  20. #include "llvm/Support/Path.h"
  21. #include "llvm/Support/YAMLParser.h"
  22. #include "llvm/Config/llvm-config.h"
  23. #include <atomic>
  24. #include <memory>
  25. // For chdir.
  26. #ifdef LLVM_ON_WIN32
  27. # include <direct.h>
  28. #else
  29. # include <unistd.h>
  30. #endif
  31. using namespace clang;
  32. using namespace clang::vfs;
  33. using namespace llvm;
  34. using llvm::sys::fs::file_status;
  35. using llvm::sys::fs::file_type;
  36. using llvm::sys::fs::perms;
  37. using llvm::sys::fs::UniqueID;
  38. Status::Status(const file_status &Status)
  39. : UID(Status.getUniqueID()), MTime(Status.getLastModificationTime()),
  40. User(Status.getUser()), Group(Status.getGroup()), Size(Status.getSize()),
  41. Type(Status.type()), Perms(Status.permissions()), IsVFSMapped(false) {}
  42. Status::Status(StringRef Name, UniqueID UID, sys::TimeValue MTime,
  43. uint32_t User, uint32_t Group, uint64_t Size, file_type Type,
  44. perms Perms)
  45. : Name(Name), UID(UID), MTime(MTime), User(User), Group(Group), Size(Size),
  46. Type(Type), Perms(Perms), IsVFSMapped(false) {}
  47. Status Status::copyWithNewName(const Status &In, StringRef NewName) {
  48. return Status(NewName, In.getUniqueID(), In.getLastModificationTime(),
  49. In.getUser(), In.getGroup(), In.getSize(), In.getType(),
  50. In.getPermissions());
  51. }
  52. Status Status::copyWithNewName(const file_status &In, StringRef NewName) {
  53. return Status(NewName, In.getUniqueID(), In.getLastModificationTime(),
  54. In.getUser(), In.getGroup(), In.getSize(), In.type(),
  55. In.permissions());
  56. }
  57. bool Status::equivalent(const Status &Other) const {
  58. return getUniqueID() == Other.getUniqueID();
  59. }
  60. bool Status::isDirectory() const {
  61. return Type == file_type::directory_file;
  62. }
  63. bool Status::isRegularFile() const {
  64. return Type == file_type::regular_file;
  65. }
  66. bool Status::isOther() const {
  67. return exists() && !isRegularFile() && !isDirectory() && !isSymlink();
  68. }
  69. bool Status::isSymlink() const {
  70. return Type == file_type::symlink_file;
  71. }
  72. bool Status::isStatusKnown() const {
  73. return Type != file_type::status_error;
  74. }
  75. bool Status::exists() const {
  76. return isStatusKnown() && Type != file_type::file_not_found;
  77. }
  78. File::~File() {}
  79. FileSystem::~FileSystem() {}
  80. ErrorOr<std::unique_ptr<MemoryBuffer>>
  81. FileSystem::getBufferForFile(const llvm::Twine &Name, int64_t FileSize,
  82. bool RequiresNullTerminator, bool IsVolatile) {
  83. auto F = openFileForRead(Name);
  84. if (!F)
  85. return F.getError();
  86. return (*F)->getBuffer(Name, FileSize, RequiresNullTerminator, IsVolatile);
  87. }
  88. std::error_code FileSystem::makeAbsolute(SmallVectorImpl<char> &Path) const {
  89. auto WorkingDir = getCurrentWorkingDirectory();
  90. if (!WorkingDir)
  91. return WorkingDir.getError();
  92. return llvm::sys::fs::make_absolute(WorkingDir.get(), Path);
  93. }
  94. bool FileSystem::exists(const Twine &Path) {
  95. auto Status = status(Path);
  96. return Status && Status->exists();
  97. }
  98. #ifndef NDEBUG
  99. static bool isTraversalComponent(StringRef Component) {
  100. return Component.equals("..") || Component.equals(".");
  101. }
  102. static bool pathHasTraversal(StringRef Path) {
  103. using namespace llvm::sys;
  104. for (StringRef Comp : llvm::make_range(path::begin(Path), path::end(Path)))
  105. if (isTraversalComponent(Comp))
  106. return true;
  107. return false;
  108. }
  109. #endif
  110. //===-----------------------------------------------------------------------===/
  111. // RealFileSystem implementation
  112. //===-----------------------------------------------------------------------===/
  113. namespace {
  114. /// \brief Wrapper around a raw file descriptor.
  115. class RealFile : public File {
  116. int FD;
  117. Status S;
  118. friend class RealFileSystem;
  119. RealFile(int FD, StringRef NewName)
  120. : FD(FD), S(NewName, {}, {}, {}, {}, {},
  121. llvm::sys::fs::file_type::status_error, {}) {
  122. assert(FD >= 0 && "Invalid or inactive file descriptor");
  123. }
  124. public:
  125. ~RealFile() override;
  126. ErrorOr<Status> status() override;
  127. ErrorOr<std::unique_ptr<MemoryBuffer>> getBuffer(const Twine &Name,
  128. int64_t FileSize,
  129. bool RequiresNullTerminator,
  130. bool IsVolatile) override;
  131. std::error_code close() override;
  132. };
  133. } // end anonymous namespace
  134. RealFile::~RealFile() { close(); }
  135. ErrorOr<Status> RealFile::status() {
  136. assert(FD != -1 && "cannot stat closed file");
  137. if (!S.isStatusKnown()) {
  138. file_status RealStatus;
  139. if (std::error_code EC = sys::fs::status(FD, RealStatus))
  140. return EC;
  141. S = Status::copyWithNewName(RealStatus, S.getName());
  142. }
  143. return S;
  144. }
  145. ErrorOr<std::unique_ptr<MemoryBuffer>>
  146. RealFile::getBuffer(const Twine &Name, int64_t FileSize,
  147. bool RequiresNullTerminator, bool IsVolatile) {
  148. assert(FD != -1 && "cannot get buffer for closed file");
  149. return MemoryBuffer::getOpenFile(FD, Name, FileSize, RequiresNullTerminator,
  150. IsVolatile);
  151. }
  152. // FIXME: This is terrible, we need this for ::close.
  153. #if !defined(_MSC_VER) && !defined(__MINGW32__)
  154. #include <unistd.h>
  155. #include <sys/uio.h>
  156. #else
  157. #include <io.h>
  158. #ifndef S_ISFIFO
  159. #define S_ISFIFO(x) (0)
  160. #endif
  161. #endif
  162. std::error_code RealFile::close() {
  163. if (::close(FD))
  164. return std::error_code(errno, std::generic_category());
  165. FD = -1;
  166. return std::error_code();
  167. }
  168. namespace {
  169. /// \brief The file system according to your operating system.
  170. class RealFileSystem : public FileSystem {
  171. public:
  172. ErrorOr<Status> status(const Twine &Path) override;
  173. ErrorOr<std::unique_ptr<File>> openFileForRead(const Twine &Path) override;
  174. directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override;
  175. llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override;
  176. std::error_code setCurrentWorkingDirectory(const Twine &Path) override;
  177. };
  178. } // end anonymous namespace
  179. ErrorOr<Status> RealFileSystem::status(const Twine &Path) {
  180. sys::fs::file_status RealStatus;
  181. if (std::error_code EC = sys::fs::status(Path, RealStatus))
  182. return EC;
  183. return Status::copyWithNewName(RealStatus, Path.str());
  184. }
  185. ErrorOr<std::unique_ptr<File>>
  186. RealFileSystem::openFileForRead(const Twine &Name) {
  187. int FD;
  188. if (std::error_code EC = sys::fs::openFileForRead(Name, FD))
  189. return EC;
  190. return std::unique_ptr<File>(new RealFile(FD, Name.str()));
  191. }
  192. llvm::ErrorOr<std::string> RealFileSystem::getCurrentWorkingDirectory() const {
  193. SmallString<256> Dir;
  194. if (std::error_code EC = llvm::sys::fs::current_path(Dir))
  195. return EC;
  196. return Dir.str().str();
  197. }
  198. std::error_code RealFileSystem::setCurrentWorkingDirectory(const Twine &Path) {
  199. // FIXME: chdir is thread hostile; on the other hand, creating the same
  200. // behavior as chdir is complex: chdir resolves the path once, thus
  201. // guaranteeing that all subsequent relative path operations work
  202. // on the same path the original chdir resulted in. This makes a
  203. // difference for example on network filesystems, where symlinks might be
  204. // switched during runtime of the tool. Fixing this depends on having a
  205. // file system abstraction that allows openat() style interactions.
  206. SmallString<256> Storage;
  207. StringRef Dir = Path.toNullTerminatedStringRef(Storage);
  208. if (int Err = ::chdir(Dir.data()))
  209. return std::error_code(Err, std::generic_category());
  210. return std::error_code();
  211. }
  212. IntrusiveRefCntPtr<FileSystem> vfs::getRealFileSystem() {
  213. static IntrusiveRefCntPtr<FileSystem> FS = new RealFileSystem();
  214. return FS;
  215. }
  216. namespace {
  217. class RealFSDirIter : public clang::vfs::detail::DirIterImpl {
  218. std::string Path;
  219. llvm::sys::fs::directory_iterator Iter;
  220. public:
  221. RealFSDirIter(const Twine &_Path, std::error_code &EC)
  222. : Path(_Path.str()), Iter(Path, EC) {
  223. if (!EC && Iter != llvm::sys::fs::directory_iterator()) {
  224. llvm::sys::fs::file_status S;
  225. EC = Iter->status(S);
  226. if (!EC)
  227. CurrentEntry = Status::copyWithNewName(S, Iter->path());
  228. }
  229. }
  230. std::error_code increment() override {
  231. std::error_code EC;
  232. Iter.increment(EC);
  233. if (EC) {
  234. return EC;
  235. } else if (Iter == llvm::sys::fs::directory_iterator()) {
  236. CurrentEntry = Status();
  237. } else {
  238. llvm::sys::fs::file_status S;
  239. EC = Iter->status(S);
  240. CurrentEntry = Status::copyWithNewName(S, Iter->path());
  241. }
  242. return EC;
  243. }
  244. };
  245. }
  246. directory_iterator RealFileSystem::dir_begin(const Twine &Dir,
  247. std::error_code &EC) {
  248. return directory_iterator(std::make_shared<RealFSDirIter>(Dir, EC));
  249. }
  250. //===-----------------------------------------------------------------------===/
  251. // OverlayFileSystem implementation
  252. //===-----------------------------------------------------------------------===/
  253. OverlayFileSystem::OverlayFileSystem(IntrusiveRefCntPtr<FileSystem> BaseFS) {
  254. FSList.push_back(BaseFS);
  255. }
  256. void OverlayFileSystem::pushOverlay(IntrusiveRefCntPtr<FileSystem> FS) {
  257. FSList.push_back(FS);
  258. // Synchronize added file systems by duplicating the working directory from
  259. // the first one in the list.
  260. FS->setCurrentWorkingDirectory(getCurrentWorkingDirectory().get());
  261. }
  262. ErrorOr<Status> OverlayFileSystem::status(const Twine &Path) {
  263. // FIXME: handle symlinks that cross file systems
  264. for (iterator I = overlays_begin(), E = overlays_end(); I != E; ++I) {
  265. ErrorOr<Status> Status = (*I)->status(Path);
  266. if (Status || Status.getError() != llvm::errc::no_such_file_or_directory)
  267. return Status;
  268. }
  269. return make_error_code(llvm::errc::no_such_file_or_directory);
  270. }
  271. ErrorOr<std::unique_ptr<File>>
  272. OverlayFileSystem::openFileForRead(const llvm::Twine &Path) {
  273. // FIXME: handle symlinks that cross file systems
  274. for (iterator I = overlays_begin(), E = overlays_end(); I != E; ++I) {
  275. auto Result = (*I)->openFileForRead(Path);
  276. if (Result || Result.getError() != llvm::errc::no_such_file_or_directory)
  277. return Result;
  278. }
  279. return make_error_code(llvm::errc::no_such_file_or_directory);
  280. }
  281. llvm::ErrorOr<std::string>
  282. OverlayFileSystem::getCurrentWorkingDirectory() const {
  283. // All file systems are synchronized, just take the first working directory.
  284. return FSList.front()->getCurrentWorkingDirectory();
  285. }
  286. std::error_code
  287. OverlayFileSystem::setCurrentWorkingDirectory(const Twine &Path) {
  288. for (auto &FS : FSList)
  289. if (std::error_code EC = FS->setCurrentWorkingDirectory(Path))
  290. return EC;
  291. return std::error_code();
  292. }
  293. clang::vfs::detail::DirIterImpl::~DirIterImpl() { }
  294. namespace {
  295. class OverlayFSDirIterImpl : public clang::vfs::detail::DirIterImpl {
  296. OverlayFileSystem &Overlays;
  297. std::string Path;
  298. OverlayFileSystem::iterator CurrentFS;
  299. directory_iterator CurrentDirIter;
  300. llvm::StringSet<> SeenNames;
  301. std::error_code incrementFS() {
  302. assert(CurrentFS != Overlays.overlays_end() && "incrementing past end");
  303. ++CurrentFS;
  304. for (auto E = Overlays.overlays_end(); CurrentFS != E; ++CurrentFS) {
  305. std::error_code EC;
  306. CurrentDirIter = (*CurrentFS)->dir_begin(Path, EC);
  307. if (EC && EC != errc::no_such_file_or_directory)
  308. return EC;
  309. if (CurrentDirIter != directory_iterator())
  310. break; // found
  311. }
  312. return std::error_code();
  313. }
  314. std::error_code incrementDirIter(bool IsFirstTime) {
  315. assert((IsFirstTime || CurrentDirIter != directory_iterator()) &&
  316. "incrementing past end");
  317. std::error_code EC;
  318. if (!IsFirstTime)
  319. CurrentDirIter.increment(EC);
  320. if (!EC && CurrentDirIter == directory_iterator())
  321. EC = incrementFS();
  322. return EC;
  323. }
  324. std::error_code incrementImpl(bool IsFirstTime) {
  325. while (true) {
  326. std::error_code EC = incrementDirIter(IsFirstTime);
  327. if (EC || CurrentDirIter == directory_iterator()) {
  328. CurrentEntry = Status();
  329. return EC;
  330. }
  331. CurrentEntry = *CurrentDirIter;
  332. StringRef Name = llvm::sys::path::filename(CurrentEntry.getName());
  333. if (SeenNames.insert(Name).second)
  334. return EC; // name not seen before
  335. }
  336. llvm_unreachable("returned above");
  337. }
  338. public:
  339. OverlayFSDirIterImpl(const Twine &Path, OverlayFileSystem &FS,
  340. std::error_code &EC)
  341. : Overlays(FS), Path(Path.str()), CurrentFS(Overlays.overlays_begin()) {
  342. CurrentDirIter = (*CurrentFS)->dir_begin(Path, EC);
  343. EC = incrementImpl(true);
  344. }
  345. std::error_code increment() override { return incrementImpl(false); }
  346. };
  347. } // end anonymous namespace
  348. directory_iterator OverlayFileSystem::dir_begin(const Twine &Dir,
  349. std::error_code &EC) {
  350. return directory_iterator(
  351. std::make_shared<OverlayFSDirIterImpl>(Dir, *this, EC));
  352. }
  353. namespace clang {
  354. namespace vfs {
  355. namespace detail {
  356. enum InMemoryNodeKind { IME_File, IME_Directory };
  357. /// The in memory file system is a tree of Nodes. Every node can either be a
  358. /// file or a directory.
  359. class InMemoryNode {
  360. Status Stat;
  361. InMemoryNodeKind Kind;
  362. public:
  363. InMemoryNode(Status Stat, InMemoryNodeKind Kind)
  364. : Stat(std::move(Stat)), Kind(Kind) {}
  365. virtual ~InMemoryNode() {}
  366. const Status &getStatus() const { return Stat; }
  367. InMemoryNodeKind getKind() const { return Kind; }
  368. virtual std::string toString(unsigned Indent) const = 0;
  369. };
  370. namespace {
  371. class InMemoryFile : public InMemoryNode {
  372. std::unique_ptr<llvm::MemoryBuffer> Buffer;
  373. public:
  374. InMemoryFile(Status Stat, std::unique_ptr<llvm::MemoryBuffer> Buffer)
  375. : InMemoryNode(std::move(Stat), IME_File), Buffer(std::move(Buffer)) {}
  376. llvm::MemoryBuffer *getBuffer() { return Buffer.get(); }
  377. std::string toString(unsigned Indent) const override {
  378. return (std::string(Indent, ' ') + getStatus().getName() + "\n").str();
  379. }
  380. static bool classof(const InMemoryNode *N) {
  381. return N->getKind() == IME_File;
  382. }
  383. };
  384. /// Adapt a InMemoryFile for VFS' File interface.
  385. class InMemoryFileAdaptor : public File {
  386. InMemoryFile &Node;
  387. public:
  388. explicit InMemoryFileAdaptor(InMemoryFile &Node) : Node(Node) {}
  389. llvm::ErrorOr<Status> status() override { return Node.getStatus(); }
  390. llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
  391. getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
  392. bool IsVolatile) override {
  393. llvm::MemoryBuffer *Buf = Node.getBuffer();
  394. return llvm::MemoryBuffer::getMemBuffer(
  395. Buf->getBuffer(), Buf->getBufferIdentifier(), RequiresNullTerminator);
  396. }
  397. std::error_code close() override { return std::error_code(); }
  398. };
  399. } // end anonymous namespace
  400. class InMemoryDirectory : public InMemoryNode {
  401. std::map<std::string, std::unique_ptr<InMemoryNode>> Entries;
  402. public:
  403. InMemoryDirectory(Status Stat)
  404. : InMemoryNode(std::move(Stat), IME_Directory) {}
  405. InMemoryNode *getChild(StringRef Name) {
  406. auto I = Entries.find(Name);
  407. if (I != Entries.end())
  408. return I->second.get();
  409. return nullptr;
  410. }
  411. InMemoryNode *addChild(StringRef Name, std::unique_ptr<InMemoryNode> Child) {
  412. return Entries.insert(make_pair(Name, std::move(Child)))
  413. .first->second.get();
  414. }
  415. typedef decltype(Entries)::const_iterator const_iterator;
  416. const_iterator begin() const { return Entries.begin(); }
  417. const_iterator end() const { return Entries.end(); }
  418. std::string toString(unsigned Indent) const override {
  419. std::string Result =
  420. (std::string(Indent, ' ') + getStatus().getName() + "\n").str();
  421. for (const auto &Entry : Entries) {
  422. Result += Entry.second->toString(Indent + 2);
  423. }
  424. return Result;
  425. }
  426. static bool classof(const InMemoryNode *N) {
  427. return N->getKind() == IME_Directory;
  428. }
  429. };
  430. }
  431. InMemoryFileSystem::InMemoryFileSystem(bool UseNormalizedPaths)
  432. : Root(new detail::InMemoryDirectory(
  433. Status("", getNextVirtualUniqueID(), llvm::sys::TimeValue::MinTime(),
  434. 0, 0, 0, llvm::sys::fs::file_type::directory_file,
  435. llvm::sys::fs::perms::all_all))),
  436. UseNormalizedPaths(UseNormalizedPaths) {}
  437. InMemoryFileSystem::~InMemoryFileSystem() {}
  438. std::string InMemoryFileSystem::toString() const {
  439. return Root->toString(/*Indent=*/0);
  440. }
  441. bool InMemoryFileSystem::addFile(const Twine &P, time_t ModificationTime,
  442. std::unique_ptr<llvm::MemoryBuffer> Buffer) {
  443. SmallString<128> Path;
  444. P.toVector(Path);
  445. // Fix up relative paths. This just prepends the current working directory.
  446. std::error_code EC = makeAbsolute(Path);
  447. assert(!EC);
  448. (void)EC;
  449. if (useNormalizedPaths())
  450. llvm::sys::path::remove_dots(Path, /*remove_dot_dot=*/true);
  451. if (Path.empty())
  452. return false;
  453. detail::InMemoryDirectory *Dir = Root.get();
  454. auto I = llvm::sys::path::begin(Path), E = llvm::sys::path::end(Path);
  455. while (true) {
  456. StringRef Name = *I;
  457. detail::InMemoryNode *Node = Dir->getChild(Name);
  458. ++I;
  459. if (!Node) {
  460. if (I == E) {
  461. // End of the path, create a new file.
  462. // FIXME: expose the status details in the interface.
  463. Status Stat(P.str(), getNextVirtualUniqueID(),
  464. llvm::sys::TimeValue(ModificationTime, 0), 0, 0,
  465. Buffer->getBufferSize(),
  466. llvm::sys::fs::file_type::regular_file,
  467. llvm::sys::fs::all_all);
  468. Dir->addChild(Name, llvm::make_unique<detail::InMemoryFile>(
  469. std::move(Stat), std::move(Buffer)));
  470. return true;
  471. }
  472. // Create a new directory. Use the path up to here.
  473. // FIXME: expose the status details in the interface.
  474. Status Stat(
  475. StringRef(Path.str().begin(), Name.end() - Path.str().begin()),
  476. getNextVirtualUniqueID(), llvm::sys::TimeValue(ModificationTime, 0),
  477. 0, 0, Buffer->getBufferSize(),
  478. llvm::sys::fs::file_type::directory_file, llvm::sys::fs::all_all);
  479. Dir = cast<detail::InMemoryDirectory>(Dir->addChild(
  480. Name, llvm::make_unique<detail::InMemoryDirectory>(std::move(Stat))));
  481. continue;
  482. }
  483. if (auto *NewDir = dyn_cast<detail::InMemoryDirectory>(Node)) {
  484. Dir = NewDir;
  485. } else {
  486. assert(isa<detail::InMemoryFile>(Node) &&
  487. "Must be either file or directory!");
  488. // Trying to insert a directory in place of a file.
  489. if (I != E)
  490. return false;
  491. // Return false only if the new file is different from the existing one.
  492. return cast<detail::InMemoryFile>(Node)->getBuffer()->getBuffer() ==
  493. Buffer->getBuffer();
  494. }
  495. }
  496. }
  497. bool InMemoryFileSystem::addFileNoOwn(const Twine &P, time_t ModificationTime,
  498. llvm::MemoryBuffer *Buffer) {
  499. return addFile(P, ModificationTime,
  500. llvm::MemoryBuffer::getMemBuffer(
  501. Buffer->getBuffer(), Buffer->getBufferIdentifier()));
  502. }
  503. static ErrorOr<detail::InMemoryNode *>
  504. lookupInMemoryNode(const InMemoryFileSystem &FS, detail::InMemoryDirectory *Dir,
  505. const Twine &P) {
  506. SmallString<128> Path;
  507. P.toVector(Path);
  508. // Fix up relative paths. This just prepends the current working directory.
  509. std::error_code EC = FS.makeAbsolute(Path);
  510. assert(!EC);
  511. (void)EC;
  512. if (FS.useNormalizedPaths())
  513. llvm::sys::path::remove_dots(Path, /*remove_dot_dot=*/true);
  514. if (Path.empty())
  515. return Dir;
  516. auto I = llvm::sys::path::begin(Path), E = llvm::sys::path::end(Path);
  517. while (true) {
  518. detail::InMemoryNode *Node = Dir->getChild(*I);
  519. ++I;
  520. if (!Node)
  521. return errc::no_such_file_or_directory;
  522. // Return the file if it's at the end of the path.
  523. if (auto File = dyn_cast<detail::InMemoryFile>(Node)) {
  524. if (I == E)
  525. return File;
  526. return errc::no_such_file_or_directory;
  527. }
  528. // Traverse directories.
  529. Dir = cast<detail::InMemoryDirectory>(Node);
  530. if (I == E)
  531. return Dir;
  532. }
  533. }
  534. llvm::ErrorOr<Status> InMemoryFileSystem::status(const Twine &Path) {
  535. auto Node = lookupInMemoryNode(*this, Root.get(), Path);
  536. if (Node)
  537. return (*Node)->getStatus();
  538. return Node.getError();
  539. }
  540. llvm::ErrorOr<std::unique_ptr<File>>
  541. InMemoryFileSystem::openFileForRead(const Twine &Path) {
  542. auto Node = lookupInMemoryNode(*this, Root.get(), Path);
  543. if (!Node)
  544. return Node.getError();
  545. // When we have a file provide a heap-allocated wrapper for the memory buffer
  546. // to match the ownership semantics for File.
  547. if (auto *F = dyn_cast<detail::InMemoryFile>(*Node))
  548. return std::unique_ptr<File>(new detail::InMemoryFileAdaptor(*F));
  549. // FIXME: errc::not_a_file?
  550. return make_error_code(llvm::errc::invalid_argument);
  551. }
  552. namespace {
  553. /// Adaptor from InMemoryDir::iterator to directory_iterator.
  554. class InMemoryDirIterator : public clang::vfs::detail::DirIterImpl {
  555. detail::InMemoryDirectory::const_iterator I;
  556. detail::InMemoryDirectory::const_iterator E;
  557. public:
  558. InMemoryDirIterator() {}
  559. explicit InMemoryDirIterator(detail::InMemoryDirectory &Dir)
  560. : I(Dir.begin()), E(Dir.end()) {
  561. if (I != E)
  562. CurrentEntry = I->second->getStatus();
  563. }
  564. std::error_code increment() override {
  565. ++I;
  566. // When we're at the end, make CurrentEntry invalid and DirIterImpl will do
  567. // the rest.
  568. CurrentEntry = I != E ? I->second->getStatus() : Status();
  569. return std::error_code();
  570. }
  571. };
  572. } // end anonymous namespace
  573. directory_iterator InMemoryFileSystem::dir_begin(const Twine &Dir,
  574. std::error_code &EC) {
  575. auto Node = lookupInMemoryNode(*this, Root.get(), Dir);
  576. if (!Node) {
  577. EC = Node.getError();
  578. return directory_iterator(std::make_shared<InMemoryDirIterator>());
  579. }
  580. if (auto *DirNode = dyn_cast<detail::InMemoryDirectory>(*Node))
  581. return directory_iterator(std::make_shared<InMemoryDirIterator>(*DirNode));
  582. EC = make_error_code(llvm::errc::not_a_directory);
  583. return directory_iterator(std::make_shared<InMemoryDirIterator>());
  584. }
  585. std::error_code InMemoryFileSystem::setCurrentWorkingDirectory(const Twine &P) {
  586. SmallString<128> Path;
  587. P.toVector(Path);
  588. // Fix up relative paths. This just prepends the current working directory.
  589. std::error_code EC = makeAbsolute(Path);
  590. assert(!EC);
  591. (void)EC;
  592. if (useNormalizedPaths())
  593. llvm::sys::path::remove_dots(Path, /*remove_dot_dot=*/true);
  594. if (!Path.empty())
  595. WorkingDirectory = Path.str();
  596. return std::error_code();
  597. }
  598. }
  599. }
  600. //===-----------------------------------------------------------------------===/
  601. // RedirectingFileSystem implementation
  602. //===-----------------------------------------------------------------------===/
  603. namespace {
  604. enum EntryKind {
  605. EK_Directory,
  606. EK_File
  607. };
  608. /// \brief A single file or directory in the VFS.
  609. class Entry {
  610. EntryKind Kind;
  611. std::string Name;
  612. public:
  613. virtual ~Entry();
  614. Entry(EntryKind K, StringRef Name) : Kind(K), Name(Name) {}
  615. StringRef getName() const { return Name; }
  616. EntryKind getKind() const { return Kind; }
  617. };
  618. class RedirectingDirectoryEntry : public Entry {
  619. std::vector<std::unique_ptr<Entry>> Contents;
  620. Status S;
  621. public:
  622. RedirectingDirectoryEntry(StringRef Name,
  623. std::vector<std::unique_ptr<Entry>> Contents,
  624. Status S)
  625. : Entry(EK_Directory, Name), Contents(std::move(Contents)),
  626. S(std::move(S)) {}
  627. Status getStatus() { return S; }
  628. typedef decltype(Contents)::iterator iterator;
  629. iterator contents_begin() { return Contents.begin(); }
  630. iterator contents_end() { return Contents.end(); }
  631. static bool classof(const Entry *E) { return E->getKind() == EK_Directory; }
  632. };
  633. class RedirectingFileEntry : public Entry {
  634. public:
  635. enum NameKind {
  636. NK_NotSet,
  637. NK_External,
  638. NK_Virtual
  639. };
  640. private:
  641. std::string ExternalContentsPath;
  642. NameKind UseName;
  643. public:
  644. RedirectingFileEntry(StringRef Name, StringRef ExternalContentsPath,
  645. NameKind UseName)
  646. : Entry(EK_File, Name), ExternalContentsPath(ExternalContentsPath),
  647. UseName(UseName) {}
  648. StringRef getExternalContentsPath() const { return ExternalContentsPath; }
  649. /// \brief whether to use the external path as the name for this file.
  650. bool useExternalName(bool GlobalUseExternalName) const {
  651. return UseName == NK_NotSet ? GlobalUseExternalName
  652. : (UseName == NK_External);
  653. }
  654. static bool classof(const Entry *E) { return E->getKind() == EK_File; }
  655. };
  656. class RedirectingFileSystem;
  657. class VFSFromYamlDirIterImpl : public clang::vfs::detail::DirIterImpl {
  658. std::string Dir;
  659. RedirectingFileSystem &FS;
  660. RedirectingDirectoryEntry::iterator Current, End;
  661. public:
  662. VFSFromYamlDirIterImpl(const Twine &Path, RedirectingFileSystem &FS,
  663. RedirectingDirectoryEntry::iterator Begin,
  664. RedirectingDirectoryEntry::iterator End,
  665. std::error_code &EC);
  666. std::error_code increment() override;
  667. };
  668. /// \brief A virtual file system parsed from a YAML file.
  669. ///
  670. /// Currently, this class allows creating virtual directories and mapping
  671. /// virtual file paths to existing external files, available in \c ExternalFS.
  672. ///
  673. /// The basic structure of the parsed file is:
  674. /// \verbatim
  675. /// {
  676. /// 'version': <version number>,
  677. /// <optional configuration>
  678. /// 'roots': [
  679. /// <directory entries>
  680. /// ]
  681. /// }
  682. /// \endverbatim
  683. ///
  684. /// All configuration options are optional.
  685. /// 'case-sensitive': <boolean, default=true>
  686. /// 'use-external-names': <boolean, default=true>
  687. /// 'overlay-relative': <boolean, default=false>
  688. ///
  689. /// Virtual directories are represented as
  690. /// \verbatim
  691. /// {
  692. /// 'type': 'directory',
  693. /// 'name': <string>,
  694. /// 'contents': [ <file or directory entries> ]
  695. /// }
  696. /// \endverbatim
  697. ///
  698. /// The default attributes for virtual directories are:
  699. /// \verbatim
  700. /// MTime = now() when created
  701. /// Perms = 0777
  702. /// User = Group = 0
  703. /// Size = 0
  704. /// UniqueID = unspecified unique value
  705. /// \endverbatim
  706. ///
  707. /// Re-mapped files are represented as
  708. /// \verbatim
  709. /// {
  710. /// 'type': 'file',
  711. /// 'name': <string>,
  712. /// 'use-external-name': <boolean> # Optional
  713. /// 'external-contents': <path to external file>)
  714. /// }
  715. /// \endverbatim
  716. ///
  717. /// and inherit their attributes from the external contents.
  718. ///
  719. /// Virtual file paths and external files are expected to be canonicalized
  720. /// without "..", "." and "./" in their paths.
  721. ///
  722. /// In both cases, the 'name' field may contain multiple path components (e.g.
  723. /// /path/to/file). However, any directory that contains more than one child
  724. /// must be uniquely represented by a directory entry.
  725. class RedirectingFileSystem : public vfs::FileSystem {
  726. /// The root(s) of the virtual file system.
  727. std::vector<std::unique_ptr<Entry>> Roots;
  728. /// \brief The file system to use for external references.
  729. IntrusiveRefCntPtr<FileSystem> ExternalFS;
  730. /// If IsRelativeOverlay is set, this represents the directory
  731. /// path that should be prefixed to each 'external-contents' entry
  732. /// when reading from YAML files.
  733. std::string ExternalContentsPrefixDir;
  734. /// @name Configuration
  735. /// @{
  736. /// \brief Whether to perform case-sensitive comparisons.
  737. ///
  738. /// Currently, case-insensitive matching only works correctly with ASCII.
  739. bool CaseSensitive;
  740. /// IsRelativeOverlay marks whether a IsExternalContentsPrefixDir path must
  741. /// be prefixed in every 'external-contents' when reading from YAML files.
  742. bool IsRelativeOverlay = false;
  743. /// \brief Whether to use to use the value of 'external-contents' for the
  744. /// names of files. This global value is overridable on a per-file basis.
  745. bool UseExternalNames;
  746. /// @}
  747. friend class RedirectingFileSystemParser;
  748. private:
  749. RedirectingFileSystem(IntrusiveRefCntPtr<FileSystem> ExternalFS)
  750. : ExternalFS(ExternalFS), CaseSensitive(true), UseExternalNames(true) {}
  751. /// \brief Looks up \p Path in \c Roots.
  752. ErrorOr<Entry *> lookupPath(const Twine &Path);
  753. /// \brief Looks up the path <tt>[Start, End)</tt> in \p From, possibly
  754. /// recursing into the contents of \p From if it is a directory.
  755. ErrorOr<Entry *> lookupPath(sys::path::const_iterator Start,
  756. sys::path::const_iterator End, Entry *From);
  757. /// \brief Get the status of a given an \c Entry.
  758. ErrorOr<Status> status(const Twine &Path, Entry *E);
  759. public:
  760. /// \brief Parses \p Buffer, which is expected to be in YAML format and
  761. /// returns a virtual file system representing its contents.
  762. static RedirectingFileSystem *
  763. create(std::unique_ptr<MemoryBuffer> Buffer,
  764. SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath,
  765. void *DiagContext, IntrusiveRefCntPtr<FileSystem> ExternalFS);
  766. ErrorOr<Status> status(const Twine &Path) override;
  767. ErrorOr<std::unique_ptr<File>> openFileForRead(const Twine &Path) override;
  768. llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override {
  769. return ExternalFS->getCurrentWorkingDirectory();
  770. }
  771. std::error_code setCurrentWorkingDirectory(const Twine &Path) override {
  772. return ExternalFS->setCurrentWorkingDirectory(Path);
  773. }
  774. directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override{
  775. ErrorOr<Entry *> E = lookupPath(Dir);
  776. if (!E) {
  777. EC = E.getError();
  778. return directory_iterator();
  779. }
  780. ErrorOr<Status> S = status(Dir, *E);
  781. if (!S) {
  782. EC = S.getError();
  783. return directory_iterator();
  784. }
  785. if (!S->isDirectory()) {
  786. EC = std::error_code(static_cast<int>(errc::not_a_directory),
  787. std::system_category());
  788. return directory_iterator();
  789. }
  790. auto *D = cast<RedirectingDirectoryEntry>(*E);
  791. return directory_iterator(std::make_shared<VFSFromYamlDirIterImpl>(Dir,
  792. *this, D->contents_begin(), D->contents_end(), EC));
  793. }
  794. void setExternalContentsPrefixDir(StringRef PrefixDir) {
  795. ExternalContentsPrefixDir = PrefixDir.str();
  796. }
  797. StringRef getExternalContentsPrefixDir() const {
  798. return ExternalContentsPrefixDir;
  799. }
  800. };
  801. /// \brief A helper class to hold the common YAML parsing state.
  802. class RedirectingFileSystemParser {
  803. yaml::Stream &Stream;
  804. void error(yaml::Node *N, const Twine &Msg) {
  805. Stream.printError(N, Msg);
  806. }
  807. // false on error
  808. bool parseScalarString(yaml::Node *N, StringRef &Result,
  809. SmallVectorImpl<char> &Storage) {
  810. yaml::ScalarNode *S = dyn_cast<yaml::ScalarNode>(N);
  811. if (!S) {
  812. error(N, "expected string");
  813. return false;
  814. }
  815. Result = S->getValue(Storage);
  816. return true;
  817. }
  818. // false on error
  819. bool parseScalarBool(yaml::Node *N, bool &Result) {
  820. SmallString<5> Storage;
  821. StringRef Value;
  822. if (!parseScalarString(N, Value, Storage))
  823. return false;
  824. if (Value.equals_lower("true") || Value.equals_lower("on") ||
  825. Value.equals_lower("yes") || Value == "1") {
  826. Result = true;
  827. return true;
  828. } else if (Value.equals_lower("false") || Value.equals_lower("off") ||
  829. Value.equals_lower("no") || Value == "0") {
  830. Result = false;
  831. return true;
  832. }
  833. error(N, "expected boolean value");
  834. return false;
  835. }
  836. struct KeyStatus {
  837. KeyStatus(bool Required=false) : Required(Required), Seen(false) {}
  838. bool Required;
  839. bool Seen;
  840. };
  841. typedef std::pair<StringRef, KeyStatus> KeyStatusPair;
  842. // false on error
  843. bool checkDuplicateOrUnknownKey(yaml::Node *KeyNode, StringRef Key,
  844. DenseMap<StringRef, KeyStatus> &Keys) {
  845. if (!Keys.count(Key)) {
  846. error(KeyNode, "unknown key");
  847. return false;
  848. }
  849. KeyStatus &S = Keys[Key];
  850. if (S.Seen) {
  851. error(KeyNode, Twine("duplicate key '") + Key + "'");
  852. return false;
  853. }
  854. S.Seen = true;
  855. return true;
  856. }
  857. // false on error
  858. bool checkMissingKeys(yaml::Node *Obj, DenseMap<StringRef, KeyStatus> &Keys) {
  859. for (DenseMap<StringRef, KeyStatus>::iterator I = Keys.begin(),
  860. E = Keys.end();
  861. I != E; ++I) {
  862. if (I->second.Required && !I->second.Seen) {
  863. error(Obj, Twine("missing key '") + I->first + "'");
  864. return false;
  865. }
  866. }
  867. return true;
  868. }
  869. std::unique_ptr<Entry> parseEntry(yaml::Node *N, RedirectingFileSystem *FS) {
  870. yaml::MappingNode *M = dyn_cast<yaml::MappingNode>(N);
  871. if (!M) {
  872. error(N, "expected mapping node for file or directory entry");
  873. return nullptr;
  874. }
  875. KeyStatusPair Fields[] = {
  876. KeyStatusPair("name", true),
  877. KeyStatusPair("type", true),
  878. KeyStatusPair("contents", false),
  879. KeyStatusPair("external-contents", false),
  880. KeyStatusPair("use-external-name", false),
  881. };
  882. DenseMap<StringRef, KeyStatus> Keys(std::begin(Fields), std::end(Fields));
  883. bool HasContents = false; // external or otherwise
  884. std::vector<std::unique_ptr<Entry>> EntryArrayContents;
  885. std::string ExternalContentsPath;
  886. std::string Name;
  887. auto UseExternalName = RedirectingFileEntry::NK_NotSet;
  888. EntryKind Kind;
  889. for (yaml::MappingNode::iterator I = M->begin(), E = M->end(); I != E;
  890. ++I) {
  891. StringRef Key;
  892. // Reuse the buffer for key and value, since we don't look at key after
  893. // parsing value.
  894. SmallString<256> Buffer;
  895. if (!parseScalarString(I->getKey(), Key, Buffer))
  896. return nullptr;
  897. if (!checkDuplicateOrUnknownKey(I->getKey(), Key, Keys))
  898. return nullptr;
  899. StringRef Value;
  900. if (Key == "name") {
  901. if (!parseScalarString(I->getValue(), Value, Buffer))
  902. return nullptr;
  903. SmallString<256> Path(Value);
  904. // Guarantee that old YAML files containing paths with ".." and "." are
  905. // properly canonicalized before read into the VFS.
  906. Path = sys::path::remove_leading_dotslash(Path);
  907. sys::path::remove_dots(Path, /*remove_dot_dot=*/true);
  908. Name = Path.str();
  909. } else if (Key == "type") {
  910. if (!parseScalarString(I->getValue(), Value, Buffer))
  911. return nullptr;
  912. if (Value == "file")
  913. Kind = EK_File;
  914. else if (Value == "directory")
  915. Kind = EK_Directory;
  916. else {
  917. error(I->getValue(), "unknown value for 'type'");
  918. return nullptr;
  919. }
  920. } else if (Key == "contents") {
  921. if (HasContents) {
  922. error(I->getKey(),
  923. "entry already has 'contents' or 'external-contents'");
  924. return nullptr;
  925. }
  926. HasContents = true;
  927. yaml::SequenceNode *Contents =
  928. dyn_cast<yaml::SequenceNode>(I->getValue());
  929. if (!Contents) {
  930. // FIXME: this is only for directories, what about files?
  931. error(I->getValue(), "expected array");
  932. return nullptr;
  933. }
  934. for (yaml::SequenceNode::iterator I = Contents->begin(),
  935. E = Contents->end();
  936. I != E; ++I) {
  937. if (std::unique_ptr<Entry> E = parseEntry(&*I, FS))
  938. EntryArrayContents.push_back(std::move(E));
  939. else
  940. return nullptr;
  941. }
  942. } else if (Key == "external-contents") {
  943. if (HasContents) {
  944. error(I->getKey(),
  945. "entry already has 'contents' or 'external-contents'");
  946. return nullptr;
  947. }
  948. HasContents = true;
  949. if (!parseScalarString(I->getValue(), Value, Buffer))
  950. return nullptr;
  951. SmallString<128> FullPath;
  952. if (FS->IsRelativeOverlay) {
  953. FullPath = FS->getExternalContentsPrefixDir();
  954. assert(!FullPath.empty() &&
  955. "External contents prefix directory must exist");
  956. llvm::sys::path::append(FullPath, Value);
  957. } else {
  958. FullPath = Value;
  959. }
  960. // Guarantee that old YAML files containing paths with ".." and "." are
  961. // properly canonicalized before read into the VFS.
  962. FullPath = sys::path::remove_leading_dotslash(FullPath);
  963. sys::path::remove_dots(FullPath, /*remove_dot_dot=*/true);
  964. ExternalContentsPath = FullPath.str();
  965. } else if (Key == "use-external-name") {
  966. bool Val;
  967. if (!parseScalarBool(I->getValue(), Val))
  968. return nullptr;
  969. UseExternalName = Val ? RedirectingFileEntry::NK_External
  970. : RedirectingFileEntry::NK_Virtual;
  971. } else {
  972. llvm_unreachable("key missing from Keys");
  973. }
  974. }
  975. if (Stream.failed())
  976. return nullptr;
  977. // check for missing keys
  978. if (!HasContents) {
  979. error(N, "missing key 'contents' or 'external-contents'");
  980. return nullptr;
  981. }
  982. if (!checkMissingKeys(N, Keys))
  983. return nullptr;
  984. // check invalid configuration
  985. if (Kind == EK_Directory &&
  986. UseExternalName != RedirectingFileEntry::NK_NotSet) {
  987. error(N, "'use-external-name' is not supported for directories");
  988. return nullptr;
  989. }
  990. // Remove trailing slash(es), being careful not to remove the root path
  991. StringRef Trimmed(Name);
  992. size_t RootPathLen = sys::path::root_path(Trimmed).size();
  993. while (Trimmed.size() > RootPathLen &&
  994. sys::path::is_separator(Trimmed.back()))
  995. Trimmed = Trimmed.slice(0, Trimmed.size()-1);
  996. // Get the last component
  997. StringRef LastComponent = sys::path::filename(Trimmed);
  998. std::unique_ptr<Entry> Result;
  999. switch (Kind) {
  1000. case EK_File:
  1001. Result = llvm::make_unique<RedirectingFileEntry>(
  1002. LastComponent, std::move(ExternalContentsPath), UseExternalName);
  1003. break;
  1004. case EK_Directory:
  1005. Result = llvm::make_unique<RedirectingDirectoryEntry>(
  1006. LastComponent, std::move(EntryArrayContents),
  1007. Status("", getNextVirtualUniqueID(), sys::TimeValue::now(), 0, 0, 0,
  1008. file_type::directory_file, sys::fs::all_all));
  1009. break;
  1010. }
  1011. StringRef Parent = sys::path::parent_path(Trimmed);
  1012. if (Parent.empty())
  1013. return Result;
  1014. // if 'name' contains multiple components, create implicit directory entries
  1015. for (sys::path::reverse_iterator I = sys::path::rbegin(Parent),
  1016. E = sys::path::rend(Parent);
  1017. I != E; ++I) {
  1018. std::vector<std::unique_ptr<Entry>> Entries;
  1019. Entries.push_back(std::move(Result));
  1020. Result = llvm::make_unique<RedirectingDirectoryEntry>(
  1021. *I, std::move(Entries),
  1022. Status("", getNextVirtualUniqueID(), sys::TimeValue::now(), 0, 0, 0,
  1023. file_type::directory_file, sys::fs::all_all));
  1024. }
  1025. return Result;
  1026. }
  1027. public:
  1028. RedirectingFileSystemParser(yaml::Stream &S) : Stream(S) {}
  1029. // false on error
  1030. bool parse(yaml::Node *Root, RedirectingFileSystem *FS) {
  1031. yaml::MappingNode *Top = dyn_cast<yaml::MappingNode>(Root);
  1032. if (!Top) {
  1033. error(Root, "expected mapping node");
  1034. return false;
  1035. }
  1036. KeyStatusPair Fields[] = {
  1037. KeyStatusPair("version", true),
  1038. KeyStatusPair("case-sensitive", false),
  1039. KeyStatusPair("use-external-names", false),
  1040. KeyStatusPair("overlay-relative", false),
  1041. KeyStatusPair("roots", true),
  1042. };
  1043. DenseMap<StringRef, KeyStatus> Keys(std::begin(Fields), std::end(Fields));
  1044. // Parse configuration and 'roots'
  1045. for (yaml::MappingNode::iterator I = Top->begin(), E = Top->end(); I != E;
  1046. ++I) {
  1047. SmallString<10> KeyBuffer;
  1048. StringRef Key;
  1049. if (!parseScalarString(I->getKey(), Key, KeyBuffer))
  1050. return false;
  1051. if (!checkDuplicateOrUnknownKey(I->getKey(), Key, Keys))
  1052. return false;
  1053. if (Key == "roots") {
  1054. yaml::SequenceNode *Roots = dyn_cast<yaml::SequenceNode>(I->getValue());
  1055. if (!Roots) {
  1056. error(I->getValue(), "expected array");
  1057. return false;
  1058. }
  1059. for (yaml::SequenceNode::iterator I = Roots->begin(), E = Roots->end();
  1060. I != E; ++I) {
  1061. if (std::unique_ptr<Entry> E = parseEntry(&*I, FS))
  1062. FS->Roots.push_back(std::move(E));
  1063. else
  1064. return false;
  1065. }
  1066. } else if (Key == "version") {
  1067. StringRef VersionString;
  1068. SmallString<4> Storage;
  1069. if (!parseScalarString(I->getValue(), VersionString, Storage))
  1070. return false;
  1071. int Version;
  1072. if (VersionString.getAsInteger<int>(10, Version)) {
  1073. error(I->getValue(), "expected integer");
  1074. return false;
  1075. }
  1076. if (Version < 0) {
  1077. error(I->getValue(), "invalid version number");
  1078. return false;
  1079. }
  1080. if (Version != 0) {
  1081. error(I->getValue(), "version mismatch, expected 0");
  1082. return false;
  1083. }
  1084. } else if (Key == "case-sensitive") {
  1085. if (!parseScalarBool(I->getValue(), FS->CaseSensitive))
  1086. return false;
  1087. } else if (Key == "overlay-relative") {
  1088. if (!parseScalarBool(I->getValue(), FS->IsRelativeOverlay))
  1089. return false;
  1090. } else if (Key == "use-external-names") {
  1091. if (!parseScalarBool(I->getValue(), FS->UseExternalNames))
  1092. return false;
  1093. } else {
  1094. llvm_unreachable("key missing from Keys");
  1095. }
  1096. }
  1097. if (Stream.failed())
  1098. return false;
  1099. if (!checkMissingKeys(Top, Keys))
  1100. return false;
  1101. return true;
  1102. }
  1103. };
  1104. } // end of anonymous namespace
  1105. Entry::~Entry() = default;
  1106. RedirectingFileSystem *
  1107. RedirectingFileSystem::create(std::unique_ptr<MemoryBuffer> Buffer,
  1108. SourceMgr::DiagHandlerTy DiagHandler,
  1109. StringRef YAMLFilePath, void *DiagContext,
  1110. IntrusiveRefCntPtr<FileSystem> ExternalFS) {
  1111. SourceMgr SM;
  1112. yaml::Stream Stream(Buffer->getMemBufferRef(), SM);
  1113. SM.setDiagHandler(DiagHandler, DiagContext);
  1114. yaml::document_iterator DI = Stream.begin();
  1115. yaml::Node *Root = DI->getRoot();
  1116. if (DI == Stream.end() || !Root) {
  1117. SM.PrintMessage(SMLoc(), SourceMgr::DK_Error, "expected root node");
  1118. return nullptr;
  1119. }
  1120. RedirectingFileSystemParser P(Stream);
  1121. std::unique_ptr<RedirectingFileSystem> FS(
  1122. new RedirectingFileSystem(ExternalFS));
  1123. if (!YAMLFilePath.empty()) {
  1124. // Use the YAML path from -ivfsoverlay to compute the dir to be prefixed
  1125. // to each 'external-contents' path.
  1126. //
  1127. // Example:
  1128. // -ivfsoverlay dummy.cache/vfs/vfs.yaml
  1129. // yields:
  1130. // FS->ExternalContentsPrefixDir => /<absolute_path_to>/dummy.cache/vfs
  1131. //
  1132. SmallString<256> OverlayAbsDir = sys::path::parent_path(YAMLFilePath);
  1133. std::error_code EC = llvm::sys::fs::make_absolute(OverlayAbsDir);
  1134. assert(!EC && "Overlay dir final path must be absolute");
  1135. (void)EC;
  1136. FS->setExternalContentsPrefixDir(OverlayAbsDir);
  1137. }
  1138. if (!P.parse(Root, FS.get()))
  1139. return nullptr;
  1140. return FS.release();
  1141. }
  1142. ErrorOr<Entry *> RedirectingFileSystem::lookupPath(const Twine &Path_) {
  1143. SmallString<256> Path;
  1144. Path_.toVector(Path);
  1145. // Handle relative paths
  1146. if (std::error_code EC = makeAbsolute(Path))
  1147. return EC;
  1148. // Canonicalize path by removing ".", "..", "./", etc components. This is
  1149. // a VFS request, do bot bother about symlinks in the path components
  1150. // but canonicalize in order to perform the correct entry search.
  1151. Path = sys::path::remove_leading_dotslash(Path);
  1152. sys::path::remove_dots(Path, /*remove_dot_dot=*/true);
  1153. if (Path.empty())
  1154. return make_error_code(llvm::errc::invalid_argument);
  1155. sys::path::const_iterator Start = sys::path::begin(Path);
  1156. sys::path::const_iterator End = sys::path::end(Path);
  1157. for (const std::unique_ptr<Entry> &Root : Roots) {
  1158. ErrorOr<Entry *> Result = lookupPath(Start, End, Root.get());
  1159. if (Result || Result.getError() != llvm::errc::no_such_file_or_directory)
  1160. return Result;
  1161. }
  1162. return make_error_code(llvm::errc::no_such_file_or_directory);
  1163. }
  1164. ErrorOr<Entry *>
  1165. RedirectingFileSystem::lookupPath(sys::path::const_iterator Start,
  1166. sys::path::const_iterator End, Entry *From) {
  1167. assert(!isTraversalComponent(*Start) &&
  1168. !isTraversalComponent(From->getName()) &&
  1169. "Paths should not contain traversal components");
  1170. if (CaseSensitive ? !Start->equals(From->getName())
  1171. : !Start->equals_lower(From->getName()))
  1172. // failure to match
  1173. return make_error_code(llvm::errc::no_such_file_or_directory);
  1174. ++Start;
  1175. if (Start == End) {
  1176. // Match!
  1177. return From;
  1178. }
  1179. auto *DE = dyn_cast<RedirectingDirectoryEntry>(From);
  1180. if (!DE)
  1181. return make_error_code(llvm::errc::not_a_directory);
  1182. for (const std::unique_ptr<Entry> &DirEntry :
  1183. llvm::make_range(DE->contents_begin(), DE->contents_end())) {
  1184. ErrorOr<Entry *> Result = lookupPath(Start, End, DirEntry.get());
  1185. if (Result || Result.getError() != llvm::errc::no_such_file_or_directory)
  1186. return Result;
  1187. }
  1188. return make_error_code(llvm::errc::no_such_file_or_directory);
  1189. }
  1190. static Status getRedirectedFileStatus(const Twine &Path, bool UseExternalNames,
  1191. Status ExternalStatus) {
  1192. Status S = ExternalStatus;
  1193. if (!UseExternalNames)
  1194. S = Status::copyWithNewName(S, Path.str());
  1195. S.IsVFSMapped = true;
  1196. return S;
  1197. }
  1198. ErrorOr<Status> RedirectingFileSystem::status(const Twine &Path, Entry *E) {
  1199. assert(E != nullptr);
  1200. if (auto *F = dyn_cast<RedirectingFileEntry>(E)) {
  1201. ErrorOr<Status> S = ExternalFS->status(F->getExternalContentsPath());
  1202. assert(!S || S->getName() == F->getExternalContentsPath());
  1203. if (S)
  1204. return getRedirectedFileStatus(Path, F->useExternalName(UseExternalNames),
  1205. *S);
  1206. return S;
  1207. } else { // directory
  1208. auto *DE = cast<RedirectingDirectoryEntry>(E);
  1209. return Status::copyWithNewName(DE->getStatus(), Path.str());
  1210. }
  1211. }
  1212. ErrorOr<Status> RedirectingFileSystem::status(const Twine &Path) {
  1213. ErrorOr<Entry *> Result = lookupPath(Path);
  1214. if (!Result)
  1215. return Result.getError();
  1216. return status(Path, *Result);
  1217. }
  1218. namespace {
  1219. /// Provide a file wrapper with an overriden status.
  1220. class FileWithFixedStatus : public File {
  1221. std::unique_ptr<File> InnerFile;
  1222. Status S;
  1223. public:
  1224. FileWithFixedStatus(std::unique_ptr<File> InnerFile, Status S)
  1225. : InnerFile(std::move(InnerFile)), S(S) {}
  1226. ErrorOr<Status> status() override { return S; }
  1227. ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
  1228. getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
  1229. bool IsVolatile) override {
  1230. return InnerFile->getBuffer(Name, FileSize, RequiresNullTerminator,
  1231. IsVolatile);
  1232. }
  1233. std::error_code close() override { return InnerFile->close(); }
  1234. };
  1235. } // end anonymous namespace
  1236. ErrorOr<std::unique_ptr<File>>
  1237. RedirectingFileSystem::openFileForRead(const Twine &Path) {
  1238. ErrorOr<Entry *> E = lookupPath(Path);
  1239. if (!E)
  1240. return E.getError();
  1241. auto *F = dyn_cast<RedirectingFileEntry>(*E);
  1242. if (!F) // FIXME: errc::not_a_file?
  1243. return make_error_code(llvm::errc::invalid_argument);
  1244. auto Result = ExternalFS->openFileForRead(F->getExternalContentsPath());
  1245. if (!Result)
  1246. return Result;
  1247. auto ExternalStatus = (*Result)->status();
  1248. if (!ExternalStatus)
  1249. return ExternalStatus.getError();
  1250. // FIXME: Update the status with the name and VFSMapped.
  1251. Status S = getRedirectedFileStatus(Path, F->useExternalName(UseExternalNames),
  1252. *ExternalStatus);
  1253. return std::unique_ptr<File>(
  1254. llvm::make_unique<FileWithFixedStatus>(std::move(*Result), S));
  1255. }
  1256. IntrusiveRefCntPtr<FileSystem>
  1257. vfs::getVFSFromYAML(std::unique_ptr<MemoryBuffer> Buffer,
  1258. SourceMgr::DiagHandlerTy DiagHandler,
  1259. StringRef YAMLFilePath,
  1260. void *DiagContext,
  1261. IntrusiveRefCntPtr<FileSystem> ExternalFS) {
  1262. return RedirectingFileSystem::create(std::move(Buffer), DiagHandler,
  1263. YAMLFilePath, DiagContext, ExternalFS);
  1264. }
  1265. UniqueID vfs::getNextVirtualUniqueID() {
  1266. static std::atomic<unsigned> UID;
  1267. unsigned ID = ++UID;
  1268. // The following assumes that uint64_t max will never collide with a real
  1269. // dev_t value from the OS.
  1270. return UniqueID(std::numeric_limits<uint64_t>::max(), ID);
  1271. }
  1272. void YAMLVFSWriter::addFileMapping(StringRef VirtualPath, StringRef RealPath) {
  1273. assert(sys::path::is_absolute(VirtualPath) && "virtual path not absolute");
  1274. assert(sys::path::is_absolute(RealPath) && "real path not absolute");
  1275. assert(!pathHasTraversal(VirtualPath) && "path traversal is not supported");
  1276. Mappings.emplace_back(VirtualPath, RealPath);
  1277. }
  1278. namespace {
  1279. class JSONWriter {
  1280. llvm::raw_ostream &OS;
  1281. SmallVector<StringRef, 16> DirStack;
  1282. inline unsigned getDirIndent() { return 4 * DirStack.size(); }
  1283. inline unsigned getFileIndent() { return 4 * (DirStack.size() + 1); }
  1284. bool containedIn(StringRef Parent, StringRef Path);
  1285. StringRef containedPart(StringRef Parent, StringRef Path);
  1286. void startDirectory(StringRef Path);
  1287. void endDirectory();
  1288. void writeEntry(StringRef VPath, StringRef RPath);
  1289. public:
  1290. JSONWriter(llvm::raw_ostream &OS) : OS(OS) {}
  1291. void write(ArrayRef<YAMLVFSEntry> Entries, Optional<bool> IsCaseSensitive,
  1292. Optional<bool> IsOverlayRelative, StringRef OverlayDir);
  1293. };
  1294. }
  1295. bool JSONWriter::containedIn(StringRef Parent, StringRef Path) {
  1296. using namespace llvm::sys;
  1297. // Compare each path component.
  1298. auto IParent = path::begin(Parent), EParent = path::end(Parent);
  1299. for (auto IChild = path::begin(Path), EChild = path::end(Path);
  1300. IParent != EParent && IChild != EChild; ++IParent, ++IChild) {
  1301. if (*IParent != *IChild)
  1302. return false;
  1303. }
  1304. // Have we exhausted the parent path?
  1305. return IParent == EParent;
  1306. }
  1307. StringRef JSONWriter::containedPart(StringRef Parent, StringRef Path) {
  1308. assert(!Parent.empty());
  1309. assert(containedIn(Parent, Path));
  1310. return Path.slice(Parent.size() + 1, StringRef::npos);
  1311. }
  1312. void JSONWriter::startDirectory(StringRef Path) {
  1313. StringRef Name =
  1314. DirStack.empty() ? Path : containedPart(DirStack.back(), Path);
  1315. DirStack.push_back(Path);
  1316. unsigned Indent = getDirIndent();
  1317. OS.indent(Indent) << "{\n";
  1318. OS.indent(Indent + 2) << "'type': 'directory',\n";
  1319. OS.indent(Indent + 2) << "'name': \"" << llvm::yaml::escape(Name) << "\",\n";
  1320. OS.indent(Indent + 2) << "'contents': [\n";
  1321. }
  1322. void JSONWriter::endDirectory() {
  1323. unsigned Indent = getDirIndent();
  1324. OS.indent(Indent + 2) << "]\n";
  1325. OS.indent(Indent) << "}";
  1326. DirStack.pop_back();
  1327. }
  1328. void JSONWriter::writeEntry(StringRef VPath, StringRef RPath) {
  1329. unsigned Indent = getFileIndent();
  1330. OS.indent(Indent) << "{\n";
  1331. OS.indent(Indent + 2) << "'type': 'file',\n";
  1332. OS.indent(Indent + 2) << "'name': \"" << llvm::yaml::escape(VPath) << "\",\n";
  1333. OS.indent(Indent + 2) << "'external-contents': \""
  1334. << llvm::yaml::escape(RPath) << "\"\n";
  1335. OS.indent(Indent) << "}";
  1336. }
  1337. void JSONWriter::write(ArrayRef<YAMLVFSEntry> Entries,
  1338. Optional<bool> IsCaseSensitive,
  1339. Optional<bool> IsOverlayRelative,
  1340. StringRef OverlayDir) {
  1341. using namespace llvm::sys;
  1342. OS << "{\n"
  1343. " 'version': 0,\n";
  1344. if (IsCaseSensitive.hasValue())
  1345. OS << " 'case-sensitive': '"
  1346. << (IsCaseSensitive.getValue() ? "true" : "false") << "',\n";
  1347. bool UseOverlayRelative = false;
  1348. if (IsOverlayRelative.hasValue()) {
  1349. UseOverlayRelative = IsOverlayRelative.getValue();
  1350. OS << " 'overlay-relative': '"
  1351. << (UseOverlayRelative ? "true" : "false") << "',\n";
  1352. }
  1353. OS << " 'roots': [\n";
  1354. if (!Entries.empty()) {
  1355. const YAMLVFSEntry &Entry = Entries.front();
  1356. startDirectory(path::parent_path(Entry.VPath));
  1357. StringRef RPath = Entry.RPath;
  1358. if (UseOverlayRelative) {
  1359. unsigned OverlayDirLen = OverlayDir.size();
  1360. assert(RPath.substr(0, OverlayDirLen) == OverlayDir &&
  1361. "Overlay dir must be contained in RPath");
  1362. RPath = RPath.slice(OverlayDirLen, RPath.size());
  1363. }
  1364. writeEntry(path::filename(Entry.VPath), RPath);
  1365. for (const auto &Entry : Entries.slice(1)) {
  1366. StringRef Dir = path::parent_path(Entry.VPath);
  1367. if (Dir == DirStack.back())
  1368. OS << ",\n";
  1369. else {
  1370. while (!DirStack.empty() && !containedIn(DirStack.back(), Dir)) {
  1371. OS << "\n";
  1372. endDirectory();
  1373. }
  1374. OS << ",\n";
  1375. startDirectory(Dir);
  1376. }
  1377. StringRef RPath = Entry.RPath;
  1378. if (UseOverlayRelative) {
  1379. unsigned OverlayDirLen = OverlayDir.size();
  1380. assert(RPath.substr(0, OverlayDirLen) == OverlayDir &&
  1381. "Overlay dir must be contained in RPath");
  1382. RPath = RPath.slice(OverlayDirLen, RPath.size());
  1383. }
  1384. writeEntry(path::filename(Entry.VPath), RPath);
  1385. }
  1386. while (!DirStack.empty()) {
  1387. OS << "\n";
  1388. endDirectory();
  1389. }
  1390. OS << "\n";
  1391. }
  1392. OS << " ]\n"
  1393. << "}\n";
  1394. }
  1395. void YAMLVFSWriter::write(llvm::raw_ostream &OS) {
  1396. std::sort(Mappings.begin(), Mappings.end(),
  1397. [](const YAMLVFSEntry &LHS, const YAMLVFSEntry &RHS) {
  1398. return LHS.VPath < RHS.VPath;
  1399. });
  1400. JSONWriter(OS).write(Mappings, IsCaseSensitive, IsOverlayRelative,
  1401. OverlayDir);
  1402. }
  1403. VFSFromYamlDirIterImpl::VFSFromYamlDirIterImpl(
  1404. const Twine &_Path, RedirectingFileSystem &FS,
  1405. RedirectingDirectoryEntry::iterator Begin,
  1406. RedirectingDirectoryEntry::iterator End, std::error_code &EC)
  1407. : Dir(_Path.str()), FS(FS), Current(Begin), End(End) {
  1408. if (Current != End) {
  1409. SmallString<128> PathStr(Dir);
  1410. llvm::sys::path::append(PathStr, (*Current)->getName());
  1411. llvm::ErrorOr<vfs::Status> S = FS.status(PathStr);
  1412. if (S)
  1413. CurrentEntry = *S;
  1414. else
  1415. EC = S.getError();
  1416. }
  1417. }
  1418. std::error_code VFSFromYamlDirIterImpl::increment() {
  1419. assert(Current != End && "cannot iterate past end");
  1420. if (++Current != End) {
  1421. SmallString<128> PathStr(Dir);
  1422. llvm::sys::path::append(PathStr, (*Current)->getName());
  1423. llvm::ErrorOr<vfs::Status> S = FS.status(PathStr);
  1424. if (!S)
  1425. return S.getError();
  1426. CurrentEntry = *S;
  1427. } else {
  1428. CurrentEntry = Status();
  1429. }
  1430. return std::error_code();
  1431. }
  1432. vfs::recursive_directory_iterator::recursive_directory_iterator(FileSystem &FS_,
  1433. const Twine &Path,
  1434. std::error_code &EC)
  1435. : FS(&FS_) {
  1436. directory_iterator I = FS->dir_begin(Path, EC);
  1437. if (!EC && I != directory_iterator()) {
  1438. State = std::make_shared<IterState>();
  1439. State->push(I);
  1440. }
  1441. }
  1442. vfs::recursive_directory_iterator &
  1443. recursive_directory_iterator::increment(std::error_code &EC) {
  1444. assert(FS && State && !State->empty() && "incrementing past end");
  1445. assert(State->top()->isStatusKnown() && "non-canonical end iterator");
  1446. vfs::directory_iterator End;
  1447. if (State->top()->isDirectory()) {
  1448. vfs::directory_iterator I = FS->dir_begin(State->top()->getName(), EC);
  1449. if (EC)
  1450. return *this;
  1451. if (I != End) {
  1452. State->push(I);
  1453. return *this;
  1454. }
  1455. }
  1456. while (!State->empty() && State->top().increment(EC) == End)
  1457. State->pop();
  1458. if (State->empty())
  1459. State.reset(); // end iterator
  1460. return *this;
  1461. }