[lld-link] implement -thinlto-index-only

Summary:
This implements -thinlto-index-only, -thinlto-index-only:,
and -thinlto-emit-imports-files options in lld-link. They are
analogous to their counterparts in ld.lld: -thinlto-index-only
causes us to perform ThinLTO's thin link and write index files,
but not perform code generation. -thinlto-index-only: does the
same, but also writes a text file listing the native object
files expected to be generated. -thinlto-emit-imports-files
creates a text file next to each index file, listing the files
to import from.

Reviewers: ruiu, tejohnson, pcc, rnk

Subscribers: mehdi_amini, steven_wu, dexonsmith, arphaman, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D64461

git-svn-id: https://llvm.org/svn/llvm-project/lld/trunk@365800 91177308-0d34-0410-b5e6-96231b3b80d8

COFF/Config.h

@@ -179,6 +179,9 @@ struct Configuration {
   // Used for /lldmap.
   std::string mapFile;
 
+  // Used for /thinlto-index-only:
+  llvm::StringRef thinLTOIndexOnlyArg;
+
   uint64_t imageBase = -1;
   uint64_t fileAlign = 512;
   uint64_t stackReserve = 1024 * 1024;
@@ -209,6 +212,8 @@ struct Configuration {
   bool repro = false;
   bool swaprunCD = false;
   bool swaprunNet = false;
+  bool thinLTOEmitImportsFiles;
+  bool thinLTOIndexOnly;
 };
 
 extern Configuration *config;

COFF/Driver.cpp

@@ -1441,6 +1441,11 @@ void LinkerDriver::link(ArrayRef<const char *> argsArr) {
     fatal("/manifestinput: requires /manifest:embed");
   }
 
+  config->thinLTOEmitImportsFiles = args.hasArg(OPT_thinlto_emit_imports_files);
+  config->thinLTOIndexOnly = args.hasArg(OPT_thinlto_index_only) ||
+                             args.hasArg(OPT_thinlto_index_only_arg);
+  config->thinLTOIndexOnlyArg =
+      args.getLastArgValue(OPT_thinlto_index_only_arg);
   // Handle miscellaneous boolean flags.
   config->allowBind = args.hasFlag(OPT_allowbind, OPT_allowbind_no, true);
   config->allowIsolation =
@@ -1727,8 +1732,18 @@ void LinkerDriver::link(ArrayRef<const char *> argsArr) {
     return;
 
   // Do LTO by compiling bitcode input files to a set of native COFF files then
-  // link those files.
+  // link those files (unless -thinlto-index-only was given, in which case we
+  // resolve symbols and write indices, but don't generate native code or link).
   symtab->addCombinedLTOObjects();
+
+  // If -thinlto-index-only is given, we should create only "index
+  // files" and not object files. Index file creation is already done
+  // in addCombinedLTOObject, so we are done if that's the case.
+  if (config->thinLTOIndexOnly)
+    return;
+
+  // If we generated native object files from bitcode files, this resolves
+  // references to the symbols we use from them.
   run();
 
   if (args.hasArg(OPT_include_optional)) {

COFF/LTO.cpp

@@ -18,6 +18,7 @@
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/Bitcode/BitcodeWriter.h"
 #include "llvm/IR/DiagnosticPrinter.h"
 #include "llvm/LTO/Caching.h"
 #include "llvm/LTO/Config.h"
@@ -41,7 +42,19 @@ using namespace llvm::object;
 using namespace lld;
 using namespace lld::coff;
 
-static std::unique_ptr<lto::LTO> createLTO() {
+// Creates an empty file to and returns a raw_fd_ostream to write to it.
+static std::unique_ptr<raw_fd_ostream> openFile(StringRef file) {
+  std::error_code ec;
+  auto ret =
+      llvm::make_unique<raw_fd_ostream>(file, ec, sys::fs::OpenFlags::F_None);
+  if (ec) {
+    error("cannot open " + file + ": " + ec.message());
+    return nullptr;
+  }
+  return ret;
+}
+
+static lto::Config createConfig() {
   lto::Config c;
   c.Options = initTargetOptionsFromCodeGenFlags();
 
@@ -67,14 +80,27 @@ static std::unique_ptr<lto::LTO> createLTO() {
   if (config->saveTemps)
     checkError(c.addSaveTemps(std::string(config->outputFile) + ".",
                               /*UseInputModulePath*/ true));
+  return c;
+}
+
+BitcodeCompiler::BitcodeCompiler() {
+  // Initialize indexFile.
+  if (!config->thinLTOIndexOnlyArg.empty())
+    indexFile = openFile(config->thinLTOIndexOnlyArg);
+
+  // Initialize ltoObj.
   lto::ThinBackend backend;
-  if (config->thinLTOJobs != 0)
+  if (config->thinLTOIndexOnly) {
+    auto OnIndexWrite = [&](StringRef S) { thinIndices.erase(S); };
+    backend = lto::createWriteIndexesThinBackend(
+        "", "", config->thinLTOEmitImportsFiles, indexFile.get(), OnIndexWrite);
+  } else if (config->thinLTOJobs != 0) {
     backend = lto::createInProcessThinBackend(config->thinLTOJobs);
-  return llvm::make_unique<lto::LTO>(std::move(c), backend,
-                                     config->ltoPartitions);
-}
+  }
 
-BitcodeCompiler::BitcodeCompiler() : ltoObj(createLTO()) {}
+  ltoObj = llvm::make_unique<lto::LTO>(createConfig(), backend,
+                                       config->ltoPartitions);
+}
 
 BitcodeCompiler::~BitcodeCompiler() = default;
 
@@ -86,6 +112,9 @@ void BitcodeCompiler::add(BitcodeFile &f) {
   std::vector<Symbol *> symBodies = f.getSymbols();
   std::vector<lto::SymbolResolution> resols(symBodies.size());
 
+  if (config->thinLTOIndexOnly)
+    thinIndices.insert(obj.getName());
+
   // Provide a resolution to the LTO API for each symbol.
   for (const lto::InputFile::Symbol &objSym : obj.symbols()) {
     Symbol *sym = symBodies[symNum];
@@ -129,6 +158,23 @@ std::vector<StringRef> BitcodeCompiler::compile() {
       },
       cache));
 
+  // Emit empty index files for non-indexed files
+  for (StringRef S : thinIndices) {
+    std::string Path(S);
+    openFile(Path + ".thinlto.bc");
+    if (config->thinLTOEmitImportsFiles)
+      openFile(Path + ".imports");
+  }
+
+  // ThinLTO with index only option is required to generate only the index
+  // files. After that, we exit from linker and ThinLTO backend runs in a
+  // distributed environment.
+  if (config->thinLTOIndexOnly) {
+    if (indexFile)
+      indexFile->close();
+    return {};
+  }
+
   if (!config->ltoCache.empty())
     pruneCache(config->ltoCache, config->ltoCachePolicy);
 

COFF/LTO.h

@@ -21,7 +21,9 @@
 #define LLD_COFF_LTO_H
 
 #include "lld/Common/LLVM.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/Support/raw_ostream.h"
 #include <memory>
 #include <vector>
 
@@ -49,6 +51,8 @@ private:
   std::unique_ptr<llvm::lto::LTO> ltoObj;
   std::vector<SmallString<0>> buf;
   std::vector<std::unique_ptr<MemoryBuffer>> files;
+  std::unique_ptr<llvm::raw_fd_ostream> indexFile;
+  llvm::DenseSet<StringRef> thinIndices;
 };
 }
 }

COFF/Options.td

@@ -176,6 +176,15 @@ def pdb_source_path : P<"pdbsourcepath",
                         "Base path used to make relative source file path absolute in PDB">;
 def rsp_quoting : Joined<["--"], "rsp-quoting=">,
   HelpText<"Quoting style for response files, 'windows' (default) or 'posix'">;
+def thinlto_emit_imports_files :
+    F<"thinlto-emit-imports-files">,
+    HelpText<"Emit .imports files with -thinlto-index-only">;
+def thinlto_index_only :
+    F<"thinlto-index-only">,
+    HelpText<"Instead of linking, emit ThinLTO index files">;
+def thinlto_index_only_arg : P<
+    "thinlto-index-only",
+    "-thinlto-index-only and also write native module names to file">;
 def dash_dash_version : Flag<["--"], "version">,
   HelpText<"Print version information">;
 defm threads: B<"threads",

test/COFF/Inputs/thinlto-empty.ll

@@ -0,0 +1,2 @@
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.0.24215"

test/COFF/Inputs/thinlto.ll

@@ -0,0 +1,6 @@
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.0.24215"
+
+define void @g() {
+  ret void
+}

test/COFF/thinlto-emit-imports.ll

@@ -0,0 +1,59 @@
+; REQUIRES: x86
+
+; Generate summary sections and test lld handling.
+; RUN: opt -module-summary %s -o %t1.obj
+; RUN: opt -module-summary %p/Inputs/thinlto.ll -o %t2.obj
+
+; Include a file with an empty module summary index, to ensure that the expected
+; output files are created regardless, for a distributed build system.
+; RUN: opt -module-summary %p/Inputs/thinlto-empty.ll -o %t3.obj
+
+; Ensure lld generates imports files if requested for distributed backends.
+; RUN: rm -f %t3.obj.imports %t3.obj.thinlto.bc
+; RUN: lld-link -entry:main -thinlto-index-only \
+; RUN:     -thinlto-emit-imports-files %t1.obj %t2.obj %t3.obj -out:%t4.exe
+
+; The imports file for this module contains the bitcode file for
+; Inputs/thinlto.ll
+; RUN: cat %t1.obj.imports | count 1
+; RUN: cat %t1.obj.imports | FileCheck %s --check-prefix=IMPORTS1
+; IMPORTS1: thinlto-emit-imports.ll.tmp2.obj
+
+; The imports file for Input/thinlto.ll is empty as it does not import anything.
+; RUN: cat %t2.obj.imports | count 0
+
+; The imports file for Input/thinlto_empty.ll is empty but should exist.
+; RUN: cat %t3.obj.imports | count 0
+
+; The index file should be created even for the input with an empty summary.
+; RUN: ls %t3.obj.thinlto.bc
+
+; Ensure lld generates error if unable to write to imports file.
+; RUN: rm -f %t3.obj.imports
+; RUN: touch %t3.obj.imports
+; RUN: chmod 400 %t3.obj.imports
+; RUN: not lld-link -entry:main -thinlto-index-only \
+; RUN:     -thinlto-emit-imports-files %t1.obj %t2.obj %t3.obj \
+; RUN:     -out:%t4.exe 2>&1 | FileCheck %s --check-prefix=ERR
+; ERR: cannot open {{.*}}3.obj.imports: {{P|p}}ermission denied
+
+; Ensure lld doesn't generate import files when thinlto-index-only is not enabled
+; RUN: rm -f %t1.obj.imports
+; RUN: rm -f %t2.obj.imports
+; RUN: rm -f %t3.obj.imports
+; RUN: lld-link -entry:main -thinlto-emit-imports-files \
+; RUN:     %t1.obj %t2.obj %t3.obj -out:%t4.exe
+; RUN: not ls %t1.obj.imports
+; RUN: not ls %t2.obj.imports
+; RUN: not ls %t3.obj.imports
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.0.24215"
+
+declare void @g(...)
+
+define void @main() {
+entry:
+  call void (...) @g()
+  ret void
+}

test/COFF/thinlto-index-only.ll

@@ -0,0 +1,52 @@
+; REQUIRES: x86
+
+; Basic ThinLTO tests.
+; RUN: opt -thinlto-bc %s -o %t1.obj
+; RUN: opt -thinlto-bc %p/Inputs/thinlto.ll -o %t2.obj
+; RUN: opt -thinlto-bc %p/Inputs/thinlto-empty.ll -o %t3.obj
+
+; Ensure lld generates an index and not a binary if requested.
+; RUN: rm -f %t4.exe
+; RUN: lld-link -thinlto-index-only -entry:main %t1.obj %t2.obj -out:%t4.exe
+; RUN: llvm-bcanalyzer -dump %t1.obj.thinlto.bc | FileCheck %s --check-prefix=BACKEND1
+; RUN: llvm-bcanalyzer -dump %t2.obj.thinlto.bc | FileCheck %s --check-prefix=BACKEND2
+; RUN: not test -e %t4.exe
+
+; The backend index for this module contains summaries from itself and
+; Inputs/thinlto.ll, as it imports from the latter.
+; BACKEND1: <MODULE_STRTAB_BLOCK
+; BACKEND1-NEXT: <ENTRY {{.*}} record string = '{{.*}}thinlto-index-only.ll.tmp{{.*}}.obj'
+; BACKEND1: <ENTRY {{.*}} record string = '{{.*}}thinlto-index-only.ll.tmp{{.*}}.obj'
+; BACKEND1-NOT: <ENTRY
+; BACKEND1: </MODULE_STRTAB_BLOCK
+; BACKEND1: <GLOBALVAL_SUMMARY_BLOCK
+; BACKEND1: <VERSION
+; BACKEND1: <FLAGS
+; BACKEND1: <VALUE_GUID op0={{1|2}} op1={{-5300342847281564238|-2624081020897602054}}
+; BACKEND1: <VALUE_GUID op0={{1|2}} op1={{-5300342847281564238|-2624081020897602054}}
+; BACKEND1: <COMBINED
+; BACKEND1: <COMBINED
+; BACKEND1: </GLOBALVAL_SUMMARY_BLOCK
+
+; The backend index for Input/thinlto.ll contains summaries from itself only,
+; as it does not import anything.
+; BACKEND2: <MODULE_STRTAB_BLOCK
+; BACKEND2-NEXT: <ENTRY {{.*}} record string = '{{.*}}thinlto-index-only.ll.tmp2.obj'
+; BACKEND2-NOT: <ENTRY
+; BACKEND2: </MODULE_STRTAB_BLOCK
+; BACKEND2-NEXT: <GLOBALVAL_SUMMARY_BLOCK
+; BACKEND2-NEXT: <VERSION
+; BACKEND2-NEXT: <FLAGS
+; BACKEND2-NEXT: <VALUE_GUID op0=1 op1=-5300342847281564238
+; BACKEND2-NEXT: <COMBINED
+; BACKEND2-NEXT: </GLOBALVAL_SUMMARY_BLOCK
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.0.24215"
+
+declare void @g(...)
+
+define void @main() {
+  call void (...) @g()
+  ret void
+}