Implement the writer side of the global module index.

The global module index is a "global" index for all of the module
files within a particular subdirectory in the module cache, which
keeps track of all of the "interesting" identifiers and selectors
known in each of the module files. One can perform a fast lookup in
the index to determine which module files will have more information
about entities with a particular name/selector. This information can
help eliminate redundant lookups into module files (a serious
performance problem) and help with creating auto-import/auto-include
Fix-Its.

The global module index is created or updated at the end of a
translation unit that has triggered a (re)build of a module by
scraping all of the .pcm files out of the module cache subdirectory,
so it catches everything. As with module rebuilds, we use the file
system's atomicity to synchronize.



git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@173301 91177308-0d34-0410-b5e6-96231b3b80d8

include/clang/Driver/CC1Options.td

@@ -289,6 +289,8 @@ def ast_dump_filter : Separate<["-"], "ast-dump-filter">,
   HelpText<"Use with -ast-dump or -ast-print to dump/print only AST declaration"
            " nodes having a certain substring in a qualified name. Use"
            " -ast-list to list all filterable declaration node names.">;
+def generate_module_index : Flag<["-"], "generate-module-index">,
+  HelpText<"Automatically generate or update the global module index">;
 
 let Group = Action_Group in {
 

include/clang/Frontend/CompilerInstance.h

@@ -112,7 +112,11 @@ class CompilerInstance : public ModuleLoader {
   /// \brief The result of the last module import.
   ///
   ModuleLoadResult LastModuleImportResult;
-  
+
+  /// \brief Whether we should (re)build the global module index once we
+  /// have finished with this translation unit.
+  bool BuildGlobalModuleIndex;
+
   /// \brief Holds information about the output file.
   ///
   /// If TempFilename is not empty we must rename it to Filename at the end.
@@ -186,6 +190,15 @@ public:
   /// setInvocation - Replace the current invocation.
   void setInvocation(CompilerInvocation *Value);
 
+  /// \brief Indicates whether we should (re)build the global module index.
+  bool getBuildGlobalModuleIndex() const { return BuildGlobalModuleIndex; }
+
+  /// \brief Set the flag indicating whether we should (re)build the global
+  /// module index.
+  void setBuildGlobalModuleIndex(bool Build) {
+    BuildGlobalModuleIndex = Build;
+  }
+
   /// }
   /// @name Forwarding Methods
   /// {

include/clang/Frontend/FrontendOptions.h

@@ -137,7 +137,9 @@ public:
                                            /// speed up parsing in cases you do
                                            /// not need them (e.g. with code
                                            /// completion).
-
+  unsigned GenerateModuleIndex : 1;        ///< Whether to auto-generate a
+                                           ///< global module index when a new
+                                           ///< module has been built.
   CodeCompleteOptions CodeCompleteOpts;
 
   enum {
@@ -209,8 +211,9 @@ public:
     ShowStats(false), ShowTimers(false), ShowVersion(false),
     FixWhatYouCan(false), FixOnlyWarnings(false), FixAndRecompile(false),
     FixToTemporaries(false), ARCMTMigrateEmitARCErrors(false),
-    SkipFunctionBodies(false), ARCMTAction(ARCMT_None),
-    ObjCMTAction(ObjCMT_None), ProgramAction(frontend::ParseSyntaxOnly)
+    SkipFunctionBodies(false), GenerateModuleIndex(false),
+    ARCMTAction(ARCMT_None), ObjCMTAction(ObjCMT_None),
+    ProgramAction(frontend::ParseSyntaxOnly)
   {}
 
   /// getInputKindForExtension - Return the appropriate input kind for a file

include/clang/Serialization/GlobalModuleIndex.h

@@ -0,0 +1,119 @@
+//===--- GlobalModuleIndex.h - Global Module Index --------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the GlobalModuleIndex class, which manages a global index
+// containing all of the identifiers with namespace-scope bindings attached to
+// them as well as all of the selectors that name methods, across all of the
+// modules within a given subdirectory of the module cache. It is used to
+// improve the performance of queries such as "does this identifier have any
+// top-level bindings in any module?"
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_CLANG_SERIALIZATION_GLOBAL_MODULE_INDEX_H
+#define LLVM_CLANG_SERIALIZATION_GLOBAL_MODULE_INDEX_H
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include <utility>
+
+namespace clang {
+
+class DeclarationName;
+class DirectoryEntry;
+class FileEntry;
+class FileManager;
+
+using llvm::SmallVector;
+using llvm::SmallVectorImpl;
+using llvm::StringRef;
+
+/// \brief A global index for a set of module files, providing information about
+/// the top-level identifiers and selectors within those module files.
+///
+/// The global index is an aid for name lookup into modules, offering a central
+/// place where one can look for identifiers or selectors to determine which
+/// module files contain a namespace-scope identity with that identifier or
+/// a method with that selector, respectively. This allows the client to
+/// restrict the search to only those module files known to have a binding for
+/// that identifier or selector, improving performance. Moreover, the global
+/// module index may know about module files that have not been imported, and
+/// can be queried to determine which modules the currently translation could
+/// or should load to fix a problem.
+class GlobalModuleIndex {
+  /// \brief Internal constructor. Use \c readIndex() to read an index.
+  explicit GlobalModuleIndex(FileManager &FileMgr);
+
+public:
+  /// \brief An error code returned when trying to read an index.
+  enum ErrorCode {
+    /// \brief No error occurred.
+    EC_None,
+    /// \brief The index found was out-of-date, meaning that some of the
+    /// module files are newer than the index.
+    ///
+    /// This error code is not actually fatal, because if the index is
+    /// up-to-date for any module files, it is 
+    EC_OutOfDate,
+    /// \brief No index was found.
+    EC_NotFound,
+    /// \brief Some other process is currently building the index; it is not
+    /// available yet.
+    EC_Building,
+    /// \brief There was an unspecified I/O error reading or writing the index.
+    EC_IOError
+  };
+
+  /// \brief Read a global index file for the given directory.
+  ///
+  /// \param FileMgr The file manager to use for reading files.
+  ///
+  /// \param Path The path to the specific module cache where the module files
+  /// for the intended configuration reside.
+  ///
+  /// \returns A pair containing the global module index (if it exists) and
+  /// the error code.
+  static std::pair<GlobalModuleIndex *, ErrorCode>
+  readIndex(FileManager &FileMgr, StringRef Path);
+
+  /// \brief Retrieve the set of modules that have up-to-date indexes.
+  ///
+  /// \param ModuleFiles Will be populated with the set of module files that
+  /// have been indexed.
+  void getKnownModules(SmallVectorImpl<const FileEntry *> &ModuleFiles);
+
+  /// \brief Retrieve the set of module files on which the given module file
+  /// directly depends.
+  void getModuleDependencies(const FileEntry *ModuleFile,
+                             SmallVectorImpl<const FileEntry *> &Dependencies);
+
+  /// \brief Look for all of the module files with a namespace-scope binding
+  /// for the given name, e.g., a global function, variable, or type with that
+  /// name, or declare a method with the selector.
+  ///
+  /// \param Name The name or selector to look for.
+  ///
+  /// \param DeclaringModuleFiles Will be populated with the list of module
+  /// files that declare entities with the given name.
+  ///
+  /// \returns true if any module files were found, false otherwise.
+  bool lookupName(DeclarationName Name,
+                  SmallVectorImpl<const FileEntry *> &DeclaringModuleFiles);
+
+  /// \brief Write a global index into the given
+  ///
+  /// \param FileMgr The file manager to use to load module files.
+  ///
+  /// \param Path The path to the directory containing module files, into
+  /// which the global index will be written.
+  static ErrorCode writeIndex(FileManager &FileMgr, StringRef Path);
+};
+
+}
+
+#endif

lib/Frontend/CompilerInstance.cpp

@@ -48,7 +48,8 @@
 using namespace clang;
 
 CompilerInstance::CompilerInstance()
-  : Invocation(new CompilerInvocation()), ModuleManager(0) {
+  : Invocation(new CompilerInvocation()), ModuleManager(0),
+    BuildGlobalModuleIndex(false) {
 }
 
 CompilerInstance::~CompilerInstance() {
@@ -785,6 +786,7 @@ static void compileModule(CompilerInstance &ImportingInstance,
   FrontendOptions &FrontendOpts = Invocation->getFrontendOpts();
   FrontendOpts.OutputFile = ModuleFileName.str();
   FrontendOpts.DisableFree = false;
+  FrontendOpts.GenerateModuleIndex = false;
   FrontendOpts.Inputs.clear();
   InputKind IK = getSourceInputKindFromOptions(*Invocation->getLangOpts());
 
@@ -858,6 +860,12 @@ static void compileModule(CompilerInstance &ImportingInstance,
   Instance.clearOutputFiles(/*EraseFiles=*/true);
   if (!TempModuleMapFileName.empty())
     llvm::sys::Path(TempModuleMapFileName).eraseFromDisk();
+
+  // We've rebuilt a module. If we're allowed to generate or update the global
+  // module index, record that fact in the importing compiler instance.
+  if (ImportingInstance.getFrontendOpts().GenerateModuleIndex) {
+    ImportingInstance.setBuildGlobalModuleIndex(true);
+  }
 }
 
 ModuleLoadResult

lib/Frontend/CompilerInvocation.cpp

@@ -705,7 +705,8 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
   Opts.FixAndRecompile = Args.hasArg(OPT_fixit_recompile);
   Opts.FixToTemporaries = Args.hasArg(OPT_fixit_to_temp);
   Opts.ASTDumpFilter = Args.getLastArgValue(OPT_ast_dump_filter);
-
+  Opts.GenerateModuleIndex = Args.hasArg(OPT_generate_module_index);
+  
   Opts.CodeCompleteOpts.IncludeMacros
     = Args.hasArg(OPT_code_completion_macros);
   Opts.CodeCompleteOpts.IncludeCodePatterns

lib/Frontend/FrontendAction.cpp

@@ -23,6 +23,7 @@
 #include "clang/Parse/ParseAST.h"
 #include "clang/Serialization/ASTDeserializationListener.h"
 #include "clang/Serialization/ASTReader.h"
+#include "clang/Serialization/GlobalModuleIndex.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -380,6 +381,16 @@ bool FrontendAction::Execute() {
   }
   else ExecuteAction();
 
+  // If we are supposed to rebuild the global module index, do so now unless
+  // an error occurred.
+  if (CI.getBuildGlobalModuleIndex() && CI.hasFileManager() &&
+      CI.hasPreprocessor() &&
+      (!CI.hasDiagnostics() || !CI.getDiagnostics().hasErrorOccurred())) {
+    GlobalModuleIndex::writeIndex(
+      CI.getFileManager(),
+      CI.getPreprocessor().getHeaderSearchInfo().getModuleCachePath());
+  }
+
   return true;
 }
 

lib/Serialization/ASTReader.cpp

@@ -5411,8 +5411,9 @@ namespace {
         Info->second.NameLookupTableData;
       bool FoundAnything = false;
       for (ASTDeclContextNameLookupTable::data_iterator
-	     I = LookupTable->data_begin(), E = LookupTable->data_end();
-	   I != E; ++I) {
+             I = LookupTable->data_begin(), E = LookupTable->data_end();
+           I != E;
+           ++I) {
         ASTDeclContextNameLookupTrait::data_type Data = *I;
         for (; Data.first != Data.second; ++Data.first) {
           NamedDecl *ND = This->Reader.GetLocalDeclAs<NamedDecl>(M,

lib/Serialization/CMakeLists.txt

@@ -11,6 +11,7 @@ add_clang_library(clangSerialization
   ASTWriterDecl.cpp
   ASTWriterStmt.cpp
   GeneratePCH.cpp
+  GlobalModuleIndex.cpp
   Module.cpp
   ModuleManager.cpp
   )

lib/Serialization/GlobalModuleIndex.cpp

@@ -0,0 +1,535 @@
+//===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the GlobalModuleIndex class.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ASTReaderInternals.h"
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/OnDiskHashTable.h"
+#include "clang/Serialization/ASTBitCodes.h"
+#include "clang/Serialization/GlobalModuleIndex.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Bitcode/BitstreamReader.h"
+#include "llvm/Bitcode/BitstreamWriter.h"
+#include "llvm/Support/Filesystem.h"
+#include "llvm/Support/LockFileManager.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/PathV2.h"
+using namespace clang;
+using namespace serialization;
+
+//----------------------------------------------------------------------------//
+// Shared constants
+//----------------------------------------------------------------------------//
+namespace {
+  enum {
+    /// \brief The block containing the index.
+    GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
+  };
+
+  /// \brief Describes the record types in the index.
+  enum IndexRecordTypes {
+    /// \brief Contains version information and potentially other metadata,
+    /// used to determine if we can read this global index file.
+    METADATA,
+    /// \brief Describes a module, including its file name and dependencies.
+    MODULE,
+    /// \brief The index for identifiers.
+    IDENTIFIER_INDEX
+  };
+}
+
+/// \brief The name of the global index file.
+static const char * const IndexFileName = "modules.idx";
+
+/// \brief The global index file version.
+static const unsigned CurrentVersion = 1;
+
+//----------------------------------------------------------------------------//
+// Global module index writer.
+//----------------------------------------------------------------------------//
+
+namespace {
+  /// \brief Provides information about a specific module file.
+  struct ModuleFileInfo {
+    /// \brief The numberic ID for this module file.
+    unsigned ID;
+
+    /// \brief The set of modules on which this module depends. Each entry is
+    /// a module ID.
+    SmallVector<unsigned, 4> Dependencies;
+  };
+
+  /// \brief Builder that generates the global module index file.
+  class GlobalModuleIndexBuilder {
+    FileManager &FileMgr;
+
+    /// \brief Mapping from files to module file information.
+    typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap;
+
+    /// \brief Information about each of the known module files.
+    ModuleFilesMap ModuleFiles;
+
+    /// \brief Mapping from identifiers to the list of module file IDs that
+    /// consider this identifier to be interesting.
+    typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;
+
+    /// \brief A mapping from all interesting identifiers to the set of module
+    /// files in which those identifiers are considered interesting.
+    InterestingIdentifierMap InterestingIdentifiers;
+    
+    /// \brief Write the block-info block for the global module index file.
+    void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);
+
+    /// \brief Retrieve the module file information for the given file.
+    ModuleFileInfo &getModuleFileInfo(const FileEntry *File) {
+      llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known
+        = ModuleFiles.find(File);
+      if (Known != ModuleFiles.end())
+        return Known->second;
+
+      unsigned NewID = ModuleFiles.size();
+      ModuleFileInfo &Info = ModuleFiles[File];
+      Info.ID = NewID;
+      return Info;
+    }
+
+  public:
+    explicit GlobalModuleIndexBuilder(FileManager &FileMgr) : FileMgr(FileMgr){}
+
+    /// \brief Load the contents of the given module file into the builder.
+    ///
+    /// \returns true if an error occurred, false otherwise.
+    bool loadModuleFile(const FileEntry *File);
+
+    /// \brief Write the index to the given bitstream.
+    void writeIndex(llvm::BitstreamWriter &Stream);
+  };
+}
+
+static void emitBlockID(unsigned ID, const char *Name,
+                        llvm::BitstreamWriter &Stream,
+                        SmallVectorImpl<uint64_t> &Record) {
+  Record.clear();
+  Record.push_back(ID);
+  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);
+
+  // Emit the block name if present.
+  if (Name == 0 || Name[0] == 0) return;
+  Record.clear();
+  while (*Name)
+    Record.push_back(*Name++);
+  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record);
+}
+
+static void emitRecordID(unsigned ID, const char *Name,
+                         llvm::BitstreamWriter &Stream,
+                         SmallVectorImpl<uint64_t> &Record) {
+  Record.clear();
+  Record.push_back(ID);
+  while (*Name)
+    Record.push_back(*Name++);
+  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record);
+}
+
+void
+GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
+  SmallVector<uint64_t, 64> Record;
+  Stream.EnterSubblock(llvm::bitc::BLOCKINFO_BLOCK_ID, 3);
+
+#define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record)
+#define RECORD(X) emitRecordID(X, #X, Stream, Record)
+  BLOCK(GLOBAL_INDEX_BLOCK);
+  RECORD(METADATA);
+  RECORD(MODULE);
+  RECORD(IDENTIFIER_INDEX);
+#undef RECORD
+#undef BLOCK
+
+  Stream.ExitBlock();
+}
+
+namespace clang {
+  class InterestingASTIdentifierLookupTrait
+    : public serialization::reader::ASTIdentifierLookupTraitBase {
+
+  public:
+    /// \brief The identifier and whether it is "interesting".
+    typedef std::pair<StringRef, bool> data_type;
+
+    data_type ReadData(const internal_key_type& k,
+                       const unsigned char* d,
+                       unsigned DataLen) {
+      // The first bit indicates whether this identifier is interesting.
+      // That's all we care about.
+      using namespace clang::io;
+      unsigned RawID = ReadUnalignedLE32(d);
+      bool IsInteresting = RawID & 0x01;
+      return std::make_pair(k, IsInteresting);
+    }
+  };
+}
+
+bool GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) {
+  // Open the module file.
+  OwningPtr<llvm::MemoryBuffer> Buffer;
+  Buffer.reset(FileMgr.getBufferForFile(File));
+  if (!Buffer) {
+    return true;
+  }
+
+  // Initialize the input stream
+  llvm::BitstreamReader InStreamFile;
+  llvm::BitstreamCursor InStream;
+  InStreamFile.init((const unsigned char *)Buffer->getBufferStart(),
+                  (const unsigned char *)Buffer->getBufferEnd());
+  InStream.init(InStreamFile);
+
+  // Sniff for the signature.
+  if (InStream.Read(8) != 'C' ||
+      InStream.Read(8) != 'P' ||
+      InStream.Read(8) != 'C' ||
+      InStream.Read(8) != 'H') {
+    return true;
+  }
+
+  // Record this module file and assign it a unique ID (if it doesn't have
+  // one already).
+  unsigned ID = getModuleFileInfo(File).ID;
+
+  // Search for the blocks and records we care about.
+  enum { Outer, ControlBlock, ASTBlock } State = Outer;
+  bool Done = false;
+  while (!Done) {
+    const unsigned Flags = llvm::BitstreamCursor::AF_DontPopBlockAtEnd;
+    llvm::BitstreamEntry Entry = InStream.advance(Flags);
+    switch (Entry.Kind) {
+    case llvm::BitstreamEntry::Error:
+      return true;
+
+    case llvm::BitstreamEntry::Record:
+      // In the outer state, just skip the record. We don't care.
+      if (State == Outer) {
+        InStream.skipRecord(Entry.ID);
+        continue;
+      }
+
+      // Handle potentially-interesting records below.
+      break;
+
+    case llvm::BitstreamEntry::SubBlock:
+      if (State == Outer && Entry.ID == CONTROL_BLOCK_ID) {
+        if (InStream.EnterSubBlock(CONTROL_BLOCK_ID))
+          return true;
+
+        // Found the control block.
+        State = ControlBlock;
+        continue;
+      }
+
+      if (State == Outer && Entry.ID == AST_BLOCK_ID) {
+        if (InStream.EnterSubBlock(AST_BLOCK_ID))
+          return true;
+
+        // Found the AST block.
+        State = ASTBlock;
+        continue;
+
+      }
+
+      if (InStream.SkipBlock())
+        return true;
+
+      continue;
+
+    case llvm::BitstreamEntry::EndBlock:
+      if (State == Outer) {
+        Done = true;
+      }
+      State = Outer;
+      continue;
+    }
+
+    // Read the given record.
+    SmallVector<uint64_t, 64> Record;
+    StringRef Blob;
+    unsigned Code = InStream.readRecord(Entry.ID, Record, &Blob);
+
+    // Handle module dependencies.
+    if (State == ControlBlock && Code == IMPORTS) {
+      // Load each of the imported PCH files.
+      unsigned Idx = 0, N = Record.size();
+      while (Idx < N) {
+        // Read information about the AST file.
+
+        // Skip the imported kind
+        ++Idx;
+
+        // Skip the import location
+        ++Idx;
+
+        // Retrieve the imported file name.
+        unsigned Length = Record[Idx++];
+        SmallString<128> ImportedFile(Record.begin() + Idx,
+                                      Record.begin() + Idx + Length);
+        Idx += Length;
+
+        // Find the imported module file.
+        const FileEntry *DependsOnFile = FileMgr.getFile(ImportedFile);
+        if (!DependsOnFile)
+          return true;
+
+        // Record the dependency.
+        unsigned DependsOnID = getModuleFileInfo(DependsOnFile).ID;
+        getModuleFileInfo(File).Dependencies.push_back(DependsOnID);
+      }
+
+      continue;
+    }
+
+    // Handle the identifier table
+    if (State == ASTBlock && Code == IDENTIFIER_TABLE && Record[0] > 0) {
+      typedef OnDiskChainedHashTable<InterestingASTIdentifierLookupTrait>
+        InterestingIdentifierTable;
+      llvm::OwningPtr<InterestingIdentifierTable>
+        Table(InterestingIdentifierTable::Create(
+                (const unsigned char *)Blob.data() + Record[0],
+                (const unsigned char *)Blob.data()));
+      for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
+                                                     DEnd = Table->data_end();
+           D != DEnd; ++D) {
+        std::pair<StringRef, bool> Ident = *D;
+        if (Ident.second)
+          InterestingIdentifiers[Ident.first].push_back(ID);
+      }
+    }
+
+    // FIXME: Handle the selector table.
+    
+    // We don't care about this record.
+  }
+
+  return false;
+}
+
+namespace {
+
+/// \brief Trait used to generate the identifier index as an on-disk hash
+/// table.
+class IdentifierIndexWriterTrait {
+public:
+  typedef StringRef key_type;
+  typedef StringRef key_type_ref;
+  typedef SmallVector<unsigned, 2> data_type;
+  typedef const SmallVector<unsigned, 2> &data_type_ref;
+
+  static unsigned ComputeHash(key_type_ref Key) {
+    return llvm::HashString(Key);
+  }
+
+  std::pair<unsigned,unsigned>
+  EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
+    unsigned KeyLen = Key.size();
+    unsigned DataLen = Data.size() * 4;
+    clang::io::Emit16(Out, KeyLen);
+    clang::io::Emit16(Out, DataLen);
+    return std::make_pair(KeyLen, DataLen);
+  }
+  
+  void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) {
+    Out.write(Key.data(), KeyLen);
+  }
+
+  void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
+                unsigned DataLen) {
+    for (unsigned I = 0, N = Data.size(); I != N; ++I)
+      clang::io::Emit32(Out, Data[I]);
+  }
+};
+
+}
+
+void GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
+  using namespace llvm;
+  
+  // Emit the file header.
+  Stream.Emit((unsigned)'B', 8);
+  Stream.Emit((unsigned)'C', 8);
+  Stream.Emit((unsigned)'G', 8);
+  Stream.Emit((unsigned)'I', 8);
+
+  // Write the block-info block, which describes the records in this bitcode
+  // file.
+  emitBlockInfoBlock(Stream);
+
+  Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3);
+
+  // Write the metadata.
+  SmallVector<uint64_t, 2> Record;
+  Record.push_back(CurrentVersion);
+  Stream.EmitRecord(METADATA, Record);
+
+  // Write the set of known module files.
+  for (ModuleFilesMap::iterator M = ModuleFiles.begin(),
+                                MEnd = ModuleFiles.end();
+       M != MEnd; ++M) {
+    Record.clear();
+    Record.push_back(M->second.ID);
+    Record.push_back(M->first->getSize());
+    Record.push_back(M->first->getModificationTime());
+
+    // File name
+    StringRef Name(M->first->getName());
+    Record.push_back(Name.size());
+    Record.append(Name.begin(), Name.end());
+
+    // Dependencies
+    Record.push_back(M->second.Dependencies.size());
+    Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end());
+    Stream.EmitRecord(MODULE, Record);
+  }
+
+  // Write the identifier -> module file mapping.
+  {
+    OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
+    IdentifierIndexWriterTrait Trait;
+
+    // Populate the hash table.
+    for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(),
+                                            IEnd = InterestingIdentifiers.end();
+         I != IEnd; ++I) {
+      Generator.insert(I->first(), I->second, Trait);
+    }
+    
+    // Create the on-disk hash table in a buffer.
+    SmallString<4096> IdentifierTable;
+    uint32_t BucketOffset;
+    {
+      llvm::raw_svector_ostream Out(IdentifierTable);
+      // Make sure that no bucket is at offset 0
+      clang::io::Emit32(Out, 0);
+      BucketOffset = Generator.Emit(Out, Trait);
+    }
+
+    // Create a blob abbreviation
+    BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
+    Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX));
+    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
+    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
+    unsigned IDTableAbbrev = Stream.EmitAbbrev(Abbrev);
+
+    // Write the identifier table
+    Record.clear();
+    Record.push_back(IDENTIFIER_INDEX);
+    Record.push_back(BucketOffset);
+    Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable.str());
+  }
+
+  // FIXME: Selectors.
+
+  Stream.ExitBlock();
+}
+
+GlobalModuleIndex::ErrorCode
+GlobalModuleIndex::writeIndex(FileManager &FileMgr, StringRef Path) {
+  llvm::SmallString<128> IndexPath;
+  IndexPath += Path;
+  llvm::sys::path::append(IndexPath, IndexFileName);
+
+  // Coordinate building the global index file with other processes that might
+  // try to do the same.
+  llvm::LockFileManager Locked(IndexPath);
+  switch (Locked) {
+  case llvm::LockFileManager::LFS_Error:
+    return EC_IOError;
+
+  case llvm::LockFileManager::LFS_Owned:
+    // We're responsible for building the index ourselves. Do so below.
+    break;
+
+  case llvm::LockFileManager::LFS_Shared:
+    // Someone else is responsible for building the index. We don't care
+    // when they finish, so we're done.
+    return EC_Building;
+  }
+
+  // The module index builder.
+  GlobalModuleIndexBuilder Builder(FileMgr);
+  
+  // Load each of the module files.
+  llvm::error_code EC;
+  for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
+       D != DEnd && !EC;
+       D.increment(EC)) {
+    // If this isn't a module file, we don't care.
+    if (llvm::sys::path::extension(D->path()) != ".pcm") {
+      // ... unless it's a .pcm.lock file, which indicates that someone is
+      // in the process of rebuilding a module. They'll rebuild the index
+      // at the end of that translation unit, so we don't have to.
+      if (llvm::sys::path::extension(D->path()) == ".pcm.lock")
+        return EC_Building;
+
+      continue;
+    }
+
+    // If we can't find the module file, skip it.
+    const FileEntry *ModuleFile = FileMgr.getFile(D->path());
+    if (!ModuleFile)
+      continue;
+
+    // Load this module file.
+    if (Builder.loadModuleFile(ModuleFile))
+      return EC_IOError;
+  }
+
+  // The output buffer, into which the global index will be written.
+  SmallVector<char, 16> OutputBuffer;
+  {
+    llvm::BitstreamWriter OutputStream(OutputBuffer);
+    Builder.writeIndex(OutputStream);
+  }
+
+  // Write the global index file to a temporary file.
+  llvm::SmallString<128> IndexTmpPath;
+  int TmpFD;
+  if (llvm::sys::fs::unique_file(IndexPath + "-%%%%%%%%", TmpFD, IndexTmpPath))
+    return EC_IOError;
+
+  // Open the temporary global index file for output.
+  std::string ErrorInfo;
+  llvm::raw_fd_ostream Out(IndexTmpPath.c_str(), ErrorInfo,
+                           llvm::raw_fd_ostream::F_Binary);
+  if (Out.has_error())
+    return EC_IOError;
+
+  // Write the index.
+  Out.write(OutputBuffer.data(), OutputBuffer.size());
+  Out.close();
+  if (Out.has_error())
+    return EC_IOError;
+
+  // Remove the old index file. It isn't relevant any more.
+  bool OldIndexExisted;
+  llvm::sys::fs::remove(IndexPath.str(), OldIndexExisted);
+
+  // Rename the newly-written index file to the proper name.
+  if (llvm::sys::fs::rename(IndexTmpPath.str(), IndexPath.str())) {
+    // Rename failed; just remove the 
+    llvm::sys::fs::remove(IndexTmpPath.str(), OldIndexExisted);
+    return EC_IOError;
+  }
+
+  // We're done.
+  return EC_None;
+}

test/Modules/global_index.m

@@ -0,0 +1,8 @@
+// RUN: rm -rf %t
+// RUN: %clang_cc1 -Wauto-import -fmodule-cache-path %t -fdisable-module-hash -fmodules -generate-module-index -F %S/Inputs %s -verify
+// RUN: ls %t|grep modules.idx
+// REQUIRES: shell
+
+// expected-no-diagnostics
+@import DependsOnModule;
+