Reapply [VFS] Add 'overlay-relative' field to YAML files

This reapplies r261552.

The VFS overlay mapping between virtual paths and real paths is done through
the 'external-contents' entries in YAML files, which contains hardcoded paths
to the real files.

When a module compilation crashes, headers are dumped into <name>.cache/vfs
directory and are mapped via the <name>.cache/vfs/vfs.yaml. The script
generated for reproduction uses -ivfsoverlay pointing to file to gather the
mapping between virtual paths and files inside <name>.cache/vfs. Currently, we
are only capable of reproducing such crashes in the same machine as they
happen, because of the hardcoded paths in 'external-contents'.

To be able to reproduce a crash in another machine, this patch introduces a new
option in the VFS yaml file called 'overlay-relative'. When it's equal to
'true' it means that the provided path to the YAML file through the
-ivfsoverlay option should also be used to prefix the final path for every
'external-contents'.

Example, given the invocation snippet "... -ivfsoverlay
<name>.cache/vfs/vfs.yaml" and the following entry in the yaml file:

"overlay-relative": "true",
"roots": [
...
  "type": "directory",
  "name": "/usr/include",
  "contents": [
    {
      "type": "file",
      "name": "stdio.h",
      "external-contents": "/usr/include/stdio.h"
    },
...

Here, a file manager request for virtual "/usr/include/stdio.h", that will map
into real path "/<absolute_path_to>/<name>.cache/vfs/usr/include/stdio.h.

This is a useful feature for debugging module crashes in machines other than
the one where the error happened.

Differential Revision: http://reviews.llvm.org/D17457

rdar://problem/24499339

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@263748 91177308-0d34-0410-b5e6-96231b3b80d8

include/clang/Basic/VirtualFileSystem.h

@@ -310,6 +310,7 @@ llvm::sys::fs::UniqueID getNextVirtualUniqueID();
 IntrusiveRefCntPtr<FileSystem>
 getVFSFromYAML(std::unique_ptr<llvm::MemoryBuffer> Buffer,
                llvm::SourceMgr::DiagHandlerTy DiagHandler,
+               StringRef YAMLFilePath,
                void *DiagContext = nullptr,
                IntrusiveRefCntPtr<FileSystem> ExternalFS = getRealFileSystem());
 
@@ -323,6 +324,8 @@ struct YAMLVFSEntry {
 class YAMLVFSWriter {
   std::vector<YAMLVFSEntry> Mappings;
   Optional<bool> IsCaseSensitive;
+  Optional<bool> IsOverlayRelative;
+  std::string OverlayDir;
 
 public:
   YAMLVFSWriter() {}
@@ -330,6 +333,11 @@ public:
   void setCaseSensitivity(bool CaseSensitive) {
     IsCaseSensitive = CaseSensitive;
   }
+  void setOverlayDir(StringRef OverlayDirectory) {
+    IsOverlayRelative = true;
+    OverlayDir.assign(OverlayDirectory.str());
+  }
+
   void write(llvm::raw_ostream &OS);
 };
 

lib/Basic/VirtualFileSystem.cpp

@@ -780,6 +780,7 @@ public:
 /// All configuration options are optional.
 ///   'case-sensitive': <boolean, default=true>
 ///   'use-external-names': <boolean, default=true>
+///   'overlay-relative': <boolean, default=false>
 ///
 /// Virtual directories are represented as
 /// \verbatim
@@ -819,6 +820,10 @@ class RedirectingFileSystem : public vfs::FileSystem {
   std::vector<std::unique_ptr<Entry>> Roots;
   /// \brief The file system to use for external references.
   IntrusiveRefCntPtr<FileSystem> ExternalFS;
+  /// If IsRelativeOverlay is set, this represents the directory
+  /// path that should be prefixed to each 'external-contents' entry
+  /// when reading from YAML files.
+  std::string ExternalContentsPrefixDir;
 
   /// @name Configuration
   /// @{
@@ -828,6 +833,10 @@ class RedirectingFileSystem : public vfs::FileSystem {
   /// Currently, case-insensitive matching only works correctly with ASCII.
   bool CaseSensitive;
 
+  /// IsRelativeOverlay marks whether a IsExternalContentsPrefixDir path must
+  /// be prefixed in every 'external-contents' when reading from YAML files.
+  bool IsRelativeOverlay = false;
+
   /// \brief Whether to use to use the value of 'external-contents' for the
   /// names of files.  This global value is overridable on a per-file basis.
   bool UseExternalNames;
@@ -865,8 +874,8 @@ public:
   /// returns a virtual file system representing its contents.
   static RedirectingFileSystem *
   create(std::unique_ptr<MemoryBuffer> Buffer,
-         SourceMgr::DiagHandlerTy DiagHandler, void *DiagContext,
-         IntrusiveRefCntPtr<FileSystem> ExternalFS);
+         SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath,
+         void *DiagContext, IntrusiveRefCntPtr<FileSystem> ExternalFS);
 
   ErrorOr<Status> status(const Twine &Path) override;
   ErrorOr<std::unique_ptr<File>> openFileForRead(const Twine &Path) override;
@@ -899,6 +908,15 @@ public:
     return directory_iterator(std::make_shared<VFSFromYamlDirIterImpl>(Dir,
         *this, D->contents_begin(), D->contents_end(), EC));
   }
+
+  void setExternalContentsPrefixDir(StringRef PrefixDir) {
+    ExternalContentsPrefixDir = PrefixDir.str();
+  }
+
+  StringRef getExternalContentsPrefixDir() const {
+    return ExternalContentsPrefixDir;
+  }
+
 };
 
 /// \brief A helper class to hold the common YAML parsing state.
@@ -1072,16 +1090,24 @@ class RedirectingFileSystemParser {
         HasContents = true;
         if (!parseScalarString(I->getValue(), Value, Buffer))
           return nullptr;
+
+        SmallString<256> FullPath;
+        if (FS->IsRelativeOverlay) {
+          FullPath = FS->getExternalContentsPrefixDir();
+          assert(!FullPath.empty() &&
+                 "External contents prefix directory must exist");
+          llvm::sys::path::append(FullPath, Value);
+        } else {
+          FullPath = Value;
+        }
+
         if (FS->UseCanonicalizedPaths) {
-          SmallString<256> Path(Value);
           // Guarantee that old YAML files containing paths with ".." and "."
           // are properly canonicalized before read into the VFS.
-          Path = sys::path::remove_leading_dotslash(Path);
-          sys::path::remove_dots(Path, /*remove_dot_dot=*/true);
-          ExternalContentsPath = Path.str();
-        } else {
-          ExternalContentsPath = Value;
+          FullPath = sys::path::remove_leading_dotslash(FullPath);
+          sys::path::remove_dots(FullPath, /*remove_dot_dot=*/true);
         }
+        ExternalContentsPath = FullPath.str();
       } else if (Key == "use-external-name") {
         bool Val;
         if (!parseScalarBool(I->getValue(), Val))
@@ -1167,6 +1193,7 @@ public:
       KeyStatusPair("version", true),
       KeyStatusPair("case-sensitive", false),
       KeyStatusPair("use-external-names", false),
+      KeyStatusPair("overlay-relative", false),
       KeyStatusPair("roots", true),
     };
 
@@ -1218,6 +1245,9 @@ public:
       } else if (Key == "case-sensitive") {
         if (!parseScalarBool(I->getValue(), FS->CaseSensitive))
           return false;
+      } else if (Key == "overlay-relative") {
+        if (!parseScalarBool(I->getValue(), FS->IsRelativeOverlay))
+          return false;
       } else if (Key == "use-external-names") {
         if (!parseScalarBool(I->getValue(), FS->UseExternalNames))
           return false;
@@ -1238,9 +1268,11 @@ public:
 
 Entry::~Entry() = default;
 
-RedirectingFileSystem *RedirectingFileSystem::create(
-    std::unique_ptr<MemoryBuffer> Buffer, SourceMgr::DiagHandlerTy DiagHandler,
-    void *DiagContext, IntrusiveRefCntPtr<FileSystem> ExternalFS) {
+RedirectingFileSystem *
+RedirectingFileSystem::create(std::unique_ptr<MemoryBuffer> Buffer,
+                              SourceMgr::DiagHandlerTy DiagHandler,
+                              StringRef YAMLFilePath, void *DiagContext,
+                              IntrusiveRefCntPtr<FileSystem> ExternalFS) {
 
   SourceMgr SM;
   yaml::Stream Stream(Buffer->getMemBufferRef(), SM);
@@ -1257,6 +1289,23 @@ RedirectingFileSystem *RedirectingFileSystem::create(
 
   std::unique_ptr<RedirectingFileSystem> FS(
       new RedirectingFileSystem(ExternalFS));
+
+  if (!YAMLFilePath.empty()) {
+    // Use the YAML path from -ivfsoverlay to compute the dir to be prefixed
+    // to each 'external-contents' path.
+    //
+    // Example:
+    //    -ivfsoverlay dummy.cache/vfs/vfs.yaml
+    // yields:
+    //  FS->ExternalContentsPrefixDir => /<absolute_path_to>/dummy.cache/vfs
+    //
+    SmallString<256> OverlayAbsDir = sys::path::parent_path(YAMLFilePath);
+    std::error_code EC = llvm::sys::fs::make_absolute(OverlayAbsDir);
+    assert(!EC && "Overlay dir final path must be absolute");
+    (void)EC;
+    FS->setExternalContentsPrefixDir(OverlayAbsDir);
+  }
+
   if (!P.parse(Root, FS.get()))
     return nullptr;
 
@@ -1410,10 +1459,12 @@ RedirectingFileSystem::openFileForRead(const Twine &Path) {
 
 IntrusiveRefCntPtr<FileSystem>
 vfs::getVFSFromYAML(std::unique_ptr<MemoryBuffer> Buffer,
-                    SourceMgr::DiagHandlerTy DiagHandler, void *DiagContext,
+                    SourceMgr::DiagHandlerTy DiagHandler,
+                    StringRef YAMLFilePath,
+                    void *DiagContext,
                     IntrusiveRefCntPtr<FileSystem> ExternalFS) {
   return RedirectingFileSystem::create(std::move(Buffer), DiagHandler,
-                                       DiagContext, ExternalFS);
+                                       YAMLFilePath, DiagContext, ExternalFS);
 }
 
 UniqueID vfs::getNextVirtualUniqueID() {
@@ -1445,7 +1496,8 @@ class JSONWriter {
 
 public:
   JSONWriter(llvm::raw_ostream &OS) : OS(OS) {}
-  void write(ArrayRef<YAMLVFSEntry> Entries, Optional<bool> IsCaseSensitive);
+  void write(ArrayRef<YAMLVFSEntry> Entries, Optional<bool> IsCaseSensitive,
+             Optional<bool> IsOverlayRelative, StringRef OverlayDir);
 };
 }
 
@@ -1498,7 +1550,9 @@ void JSONWriter::writeEntry(StringRef VPath, StringRef RPath) {
 }
 
 void JSONWriter::write(ArrayRef<YAMLVFSEntry> Entries,
-                       Optional<bool> IsCaseSensitive) {
+                       Optional<bool> IsCaseSensitive,
+                       Optional<bool> IsOverlayRelative,
+                       StringRef OverlayDir) {
   using namespace llvm::sys;
 
   OS << "{\n"
@@ -1506,12 +1560,27 @@ void JSONWriter::write(ArrayRef<YAMLVFSEntry> Entries,
   if (IsCaseSensitive.hasValue())
     OS << "  'case-sensitive': '"
        << (IsCaseSensitive.getValue() ? "true" : "false") << "',\n";
+  bool UseOverlayRelative = false;
+  if (IsOverlayRelative.hasValue()) {
+    UseOverlayRelative = IsOverlayRelative.getValue();
+    OS << "  'overlay-relative': '"
+       << (UseOverlayRelative ? "true" : "false") << "',\n";
+  }
   OS << "  'roots': [\n";
 
   if (!Entries.empty()) {
     const YAMLVFSEntry &Entry = Entries.front();
     startDirectory(path::parent_path(Entry.VPath));
-    writeEntry(path::filename(Entry.VPath), Entry.RPath);
+
+    StringRef RPath = Entry.RPath;
+    if (UseOverlayRelative) {
+      unsigned OverlayDirLen = OverlayDir.size();
+      assert(RPath.substr(0, OverlayDirLen) == OverlayDir &&
+             "Overlay dir must be contained in RPath");
+      RPath = RPath.slice(OverlayDirLen, RPath.size());
+    }
+
+    writeEntry(path::filename(Entry.VPath), RPath);
 
     for (const auto &Entry : Entries.slice(1)) {
       StringRef Dir = path::parent_path(Entry.VPath);
@@ -1525,7 +1594,14 @@ void JSONWriter::write(ArrayRef<YAMLVFSEntry> Entries,
         OS << ",\n";
         startDirectory(Dir);
       }
-      writeEntry(path::filename(Entry.VPath), Entry.RPath);
+      StringRef RPath = Entry.RPath;
+      if (UseOverlayRelative) {
+        unsigned OverlayDirLen = OverlayDir.size();
+        assert(RPath.substr(0, OverlayDirLen) == OverlayDir &&
+               "Overlay dir must be contained in RPath");
+        RPath = RPath.slice(OverlayDirLen, RPath.size());
+      }
+      writeEntry(path::filename(Entry.VPath), RPath);
     }
 
     while (!DirStack.empty()) {
@@ -1545,7 +1621,8 @@ void YAMLVFSWriter::write(llvm::raw_ostream &OS) {
     return LHS.VPath < RHS.VPath;
   });
 
-  JSONWriter(OS).write(Mappings, IsCaseSensitive);
+  JSONWriter(OS).write(Mappings, IsCaseSensitive, IsOverlayRelative,
+                       OverlayDir);
 }
 
 VFSFromYamlDirIterImpl::VFSFromYamlDirIterImpl(

lib/Frontend/CompilerInvocation.cpp

@@ -2378,8 +2378,8 @@ createVFSFromCompilerInvocation(const CompilerInvocation &CI,
       return IntrusiveRefCntPtr<vfs::FileSystem>();
     }
 
-    IntrusiveRefCntPtr<vfs::FileSystem> FS =
-        vfs::getVFSFromYAML(std::move(Buffer.get()), /*DiagHandler*/ nullptr);
+    IntrusiveRefCntPtr<vfs::FileSystem> FS = vfs::getVFSFromYAML(
+        std::move(Buffer.get()), /*DiagHandler*/ nullptr, File);
     if (!FS.get()) {
       Diags.Report(diag::err_invalid_vfs_overlay) << File;
       return IntrusiveRefCntPtr<vfs::FileSystem>();

lib/Frontend/ModuleDependencyCollector.cpp

@@ -50,6 +50,10 @@ void ModuleDependencyCollector::writeFileMap() {
   SmallString<256> Dest = getDest();
   llvm::sys::path::append(Dest, "vfs.yaml");
 
+  // Default to use relative overlay directories in the VFS yaml file. This
+  // allows crash reproducer scripts to work across machines.
+  VFSWriter.setOverlayDir(getDest());
+
   std::error_code EC;
   llvm::raw_fd_ostream OS(Dest, EC, llvm::sys::fs::F_Text);
   if (EC) {

test/Modules/crash-vfs-path-symlink-component.m

@@ -40,21 +40,21 @@
 // CHECKSH: "-ivfsoverlay" "crash-vfs-{{[^ ]*}}.cache/vfs/vfs.yaml"
 
 // CHECKYAML: 'type': 'directory'
-// CHECKYAML: 'name': "{{[^ ]*}}/i/usr/include",
+// CHECKYAML: 'name': "/[[PATH:.*]]/i/usr/include",
 // CHECKYAML-NEXT: 'contents': [
 // CHECKYAML-NEXT:   {
 // CHECKYAML-NEXT:     'type': 'file',
 // CHECKYAML-NEXT:     'name': "module.map",
-// CHECKYAML-NEXT:     'external-contents': "{{[^ ]*}}.cache/vfs/{{[^ ]*}}/i/usr/include/module.map"
+// CHECKYAML-NEXT:     'external-contents': "/[[PATH]]/i/usr/include/module.map"
 // CHECKYAML-NEXT:   },
 
 // CHECKYAML: 'type': 'directory'
-// CHECKYAML: 'name': "{{[^ ]*}}/i/usr",
+// CHECKYAML: 'name': "/[[PATH]]/i/usr",
 // CHECKYAML-NEXT: 'contents': [
 // CHECKYAML-NEXT:   {
 // CHECKYAML-NEXT:     'type': 'file',
 // CHECKYAML-NEXT:     'name': "module.map",
-// CHECKYAML-NEXT:     'external-contents': "{{[^ ]*}}.cache/vfs/{{[^ ]*}}/i/usr/include/module.map"
+// CHECKYAML-NEXT:     'external-contents': "/[[PATH]]/i/usr/include/module.map"
 // CHECKYAML-NEXT:   },
 
 // Test that by using the previous generated YAML file clang is able to find the

test/Modules/crash-vfs-path-traversal.m

@@ -37,12 +37,12 @@
 // CHECKSH: "-ivfsoverlay" "crash-vfs-{{[^ ]*}}.cache/vfs/vfs.yaml"
 
 // CHECKYAML: 'type': 'directory'
-// CHECKYAML: 'name': "{{[^ ]*}}/Inputs/crash-recovery/usr/include",
+// CHECKYAML: 'name': "/[[PATH:.*]]/Inputs/System/usr/include",
 // CHECKYAML-NEXT: 'contents': [
 // CHECKYAML-NEXT:   {
 // CHECKYAML-NEXT:     'type': 'file',
 // CHECKYAML-NEXT:     'name': "module.map",
-// CHECKYAML-NEXT:     'external-contents': "{{[^ ]*}}/Inputs/crash-recovery/usr/include/module.map"
+// CHECKYAML-NEXT:     'external-contents': "/[[PATH]]/Inputs/System/usr/include/module.map"
 // CHECKYAML-NEXT:   },
 
 // Replace the paths in the YAML files with relative ".." traversals
@@ -51,9 +51,10 @@
 
 // RUN: sed -e "s@usr/include@usr/include/../include@g" \
 // RUN:     %t/crash-vfs-*.cache/vfs/vfs.yaml > %t/vfs.yaml
+// RUN: cp %t/vfs.yaml %t/crash-vfs-*.cache/vfs/vfs.yaml
 // RUN: unset FORCE_CLANG_DIAGNOSTICS_CRASH
-// RUN: %clang -E %s -I %S/Inputs/crash-recovery -isysroot %/t/i/ \
-// RUN:     -ivfsoverlay %t/vfs.yaml -fmodules \
+// RUN: %clang -E %s -I %S/Inputs/System -isysroot %/t/i/ \
+// RUN:     -ivfsoverlay %t/crash-vfs-*.cache/vfs/vfs.yaml -fmodules \
 // RUN:     -fmodules-cache-path=%t/m/ 2>&1 \
 // RUN:     | FileCheck %s --check-prefix=CHECKOVERLAY
 

test/Modules/crash-vfs-relative-overlay.m

@@ -0,0 +1,56 @@
+// REQUIRES: crash-recovery, shell
+
+// FIXME: This XFAIL is cargo-culted from crash-report.c. Do we need it?
+// XFAIL: mingw32
+
+// RUN: rm -rf %t
+// RUN: mkdir -p %t/i %t/m %t
+
+// RUN: not env FORCE_CLANG_DIAGNOSTICS_CRASH= TMPDIR=%t TEMP=%t TMP=%t \
+// RUN: %clang -fsyntax-only %s -I %S/Inputs/System -isysroot %/t/i/    \
+// RUN: -fmodules -fmodules-cache-path=%t/m/ 2>&1 | FileCheck %s
+
+// RUN: FileCheck --check-prefix=CHECKSRC %s -input-file %t/crash-vfs-*.m
+// RUN: FileCheck --check-prefix=CHECKSH %s -input-file %t/crash-vfs-*.sh
+// RUN: FileCheck --check-prefix=CHECKYAML %s -input-file \
+// RUN: %t/crash-vfs-*.cache/vfs/vfs.yaml
+// RUN: find %t/crash-vfs-*.cache/vfs | \
+// RUN:   grep "Inputs/System/usr/include/stdio.h" | count 1
+
+#include "usr/include/stdio.h"
+
+// CHECK: Preprocessed source(s) and associated run script(s) are located at:
+// CHECK-NEXT: note: diagnostic msg: {{.*}}.m
+// CHECK-NEXT: note: diagnostic msg: {{.*}}.cache
+
+// CHECKSRC: @import cstd.stdio;
+
+// CHECKSH: # Crash reproducer
+// CHECKSH-NEXT: # Driver args: "-fsyntax-only"
+// CHECKSH-NEXT: # Original command: {{.*$}}
+// CHECKSH-NEXT: "-cc1"
+// CHECKSH: "-isysroot" "{{[^"]*}}/i/"
+// CHECKSH-NOT: "-fmodules-cache-path="
+// CHECKSH: "crash-vfs-{{[^ ]*}}.m"
+// CHECKSH: "-ivfsoverlay" "crash-vfs-{{[^ ]*}}.cache/vfs/vfs.yaml"
+
+// CHECKYAML: 'type': 'directory'
+// CHECKYAML: 'name': "/[[PATH:.*]]/Inputs/System/usr/include",
+// CHECKYAML-NEXT: 'contents': [
+// CHECKYAML-NEXT:   {
+// CHECKYAML-NEXT:     'type': 'file',
+// CHECKYAML-NEXT:     'name': "module.map",
+// CHECKYAML-NOT:      'external-contents': "{{[^ ]*}}.cache
+// CHECKYAML-NEXT:     'external-contents': "/[[PATH]]/Inputs/System/usr/include/module.map"
+// CHECKYAML-NEXT:   },
+
+// Test that reading the YAML file will yield the correct path after
+// the overlay dir is prefixed to access headers in .cache/vfs directory.
+
+// RUN: unset FORCE_CLANG_DIAGNOSTICS_CRASH
+// RUN: %clang -E %s -I %S/Inputs/System -isysroot %/t/i/ \
+// RUN:     -ivfsoverlay %t/crash-vfs-*.cache/vfs/vfs.yaml -fmodules \
+// RUN:     -fmodules-cache-path=%t/m/ 2>&1 \
+// RUN:     | FileCheck %s --check-prefix=CHECKOVERLAY
+
+// CHECKOVERLAY: @import cstd.stdio; /* clang -E: implicit import for "{{[^ ]*}}.cache/vfs/{{[^ ]*}}/usr/include/stdio.h"

unittests/Basic/VirtualFileSystemTest.cpp

@@ -662,7 +662,7 @@ public:
   getFromYAMLRawString(StringRef Content,
                        IntrusiveRefCntPtr<vfs::FileSystem> ExternalFS) {
     std::unique_ptr<MemoryBuffer> Buffer = MemoryBuffer::getMemBuffer(Content);
-    return getVFSFromYAML(std::move(Buffer), CountingDiagHandler, this,
+    return getVFSFromYAML(std::move(Buffer), CountingDiagHandler, "", this,
                           ExternalFS);
   }