[preprocessor] Handle correctly inclusion directives that have macro expansions, e.g

"#include MACRO(STUFF)".

-As an inclusion position for the included file, use the file location of the file where it
was included but *after* the macro expansions. We want the macro expansions to be considered
as before-in-translation-unit for everything in the included file.

-In the preprocessing record take into account that only inclusion directives can be encountered
as "out-of-order" (by comparing the start of the range which for inclusions is the hash location)
and use binary search if there is an extreme number of macro expansions in the include directive.

Fixes rdar://11111779

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@153527 91177308-0d34-0410-b5e6-96231b3b80d8

include/clang/Lex/PreprocessingRecord.h

@@ -556,7 +556,7 @@ namespace clang {
     bool isEntityInFileID(iterator PPEI, FileID FID);
 
     /// \brief Add a new preprocessed entity to this record.
-    void addPreprocessedEntity(PreprocessedEntity *Entity);
+    PPEntityID addPreprocessedEntity(PreprocessedEntity *Entity);
 
     /// \brief Returns true if this PreprocessingRecord is keeping track of
     /// conditional directives locations.

lib/Lex/PPDirectives.cpp

@@ -1453,8 +1453,12 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc,
   }
 
   // Look up the file, create a File ID for it.
-  FileID FID = SourceMgr.createFileID(File, FilenameTok.getLocation(),
-                                      FileCharacter);
+  SourceLocation IncludePos = End;
+  // If the filename string was the result of macro expansions, set the include
+  // position on the file where it will be included and after the expansions.
+  if (IncludePos.isMacroID())
+    IncludePos = SourceMgr.getExpansionRange(IncludePos).second;
+  FileID FID = SourceMgr.createFileID(File, IncludePos, FileCharacter);
   assert(!FID.isInvalid() && "Expected valid file ID");
 
   // Finally, if all is good, enter the new file!

lib/Lex/PreprocessingRecord.cpp

@@ -244,33 +244,58 @@ unsigned PreprocessingRecord::findEndLocalPreprocessedEntity(
   return I - PreprocessedEntities.begin();
 }
 
-void PreprocessingRecord::addPreprocessedEntity(PreprocessedEntity *Entity) {
+PreprocessingRecord::PPEntityID
+PreprocessingRecord::addPreprocessedEntity(PreprocessedEntity *Entity) {
   assert(Entity);
   SourceLocation BeginLoc = Entity->getSourceRange().getBegin();
-  
+
+  if (!isa<class InclusionDirective>(Entity)) {
+    assert((PreprocessedEntities.empty() ||
+            !SourceMgr.isBeforeInTranslationUnit(BeginLoc,
+                   PreprocessedEntities.back()->getSourceRange().getBegin())) &&
+           "a macro directive was encountered out-of-order");
+    PreprocessedEntities.push_back(Entity);
+    return getPPEntityID(PreprocessedEntities.size()-1, /*isLoaded=*/false);
+  }
+
   // Check normal case, this entity begin location is after the previous one.
   if (PreprocessedEntities.empty() ||
       !SourceMgr.isBeforeInTranslationUnit(BeginLoc,
                    PreprocessedEntities.back()->getSourceRange().getBegin())) {
     PreprocessedEntities.push_back(Entity);
-    return;
+    return getPPEntityID(PreprocessedEntities.size()-1, /*isLoaded=*/false);
   }
 
-  // The entity's location is not after the previous one; this can happen rarely
-  // e.g. with "#include MACRO".
-  // Iterate the entities vector in reverse until we find the right place to
-  // insert the new entity.
-  for (std::vector<PreprocessedEntity *>::iterator
-         RI = PreprocessedEntities.end(), Begin = PreprocessedEntities.begin();
-       RI != Begin; --RI) {
-    std::vector<PreprocessedEntity *>::iterator I = RI;
+  // The entity's location is not after the previous one; this can happen with
+  // include directives that form the filename using macros, e.g:
+  // "#include MACRO(STUFF)".
+
+  typedef std::vector<PreprocessedEntity *>::iterator pp_iter;
+
+  // Usually there are few macro expansions when defining the filename, do a
+  // linear search for a few entities.
+  unsigned count = 0;
+  for (pp_iter RI    = PreprocessedEntities.end(),
+               Begin = PreprocessedEntities.begin();
+       RI != Begin && count < 4; --RI, ++count) {
+    pp_iter I = RI;
     --I;
     if (!SourceMgr.isBeforeInTranslationUnit(BeginLoc,
                                            (*I)->getSourceRange().getBegin())) {
-      PreprocessedEntities.insert(RI, Entity);
-      return;
+      pp_iter insertI = PreprocessedEntities.insert(RI, Entity);
+      return getPPEntityID(insertI - PreprocessedEntities.begin(),
+                           /*isLoaded=*/false);
     }
   }
+
+  // Linear search unsuccessful. Do a binary search.
+  pp_iter I = std::upper_bound(PreprocessedEntities.begin(),
+                               PreprocessedEntities.end(),
+                               BeginLoc,
+                               PPEntityComp<&SourceRange::getBegin>(SourceMgr));
+  pp_iter insertI = PreprocessedEntities.insert(I, Entity);
+  return getPPEntityID(insertI - PreprocessedEntities.begin(),
+                       /*isLoaded=*/false);
 }
 
 void PreprocessingRecord::SetExternalSource(
@@ -351,9 +376,7 @@ void PreprocessingRecord::MacroDefined(const Token &Id,
   SourceRange R(MI->getDefinitionLoc(), MI->getDefinitionEndLoc());
   MacroDefinition *Def
       = new (*this) MacroDefinition(Id.getIdentifierInfo(), R);
-  addPreprocessedEntity(Def);
-  MacroDefinitions[MI] = getPPEntityID(PreprocessedEntities.size()-1,
-                                       /*isLoaded=*/false);
+  MacroDefinitions[MI] = addPreprocessedEntity(Def);
 }
 
 void PreprocessingRecord::MacroUndefined(const Token &Id,

test/Preprocessor/pp-record.c

@@ -2,8 +2,11 @@
 
 // http://llvm.org/PR11120
 
-#define FILE_HEADER_NAME "pp-record.h"
+#define STRINGIZE(text) STRINGIZE_I(text)
+#define STRINGIZE_I(text) #text
 
-#if defined(FILE_HEADER_NAME)
-#include FILE_HEADER_NAME
-#endif
+#define INC pp-record.h
+
+#include STRINGIZE(INC)
+
+CAKE;

test/Preprocessor/pp-record.h

@@ -1 +1,3 @@
 // Only useful for #inclusion.
+
+#define CAKE extern int is_a_lie

unittests/Basic/SourceManagerTest.cpp

@@ -176,6 +176,121 @@ TEST_F(SourceManagerTest, getMacroArgExpandedLocation) {
   EXPECT_TRUE(defLoc2.isFileID());
 }
 
+namespace {
+
+struct MacroAction {
+  SourceLocation Loc;
+  std::string Name;
+  bool isDefinition; // if false, it is expansion.
+  
+  MacroAction(SourceLocation Loc, StringRef Name, bool isDefinition)
+    : Loc(Loc), Name(Name), isDefinition(isDefinition) { }
+};
+
+class MacroTracker : public PPCallbacks {
+  std::vector<MacroAction> &Macros;
+
+public:
+  explicit MacroTracker(std::vector<MacroAction> &Macros) : Macros(Macros) { }
+  
+  virtual void MacroDefined(const Token &MacroNameTok, const MacroInfo *MI) {
+    Macros.push_back(MacroAction(MI->getDefinitionLoc(),
+                                 MacroNameTok.getIdentifierInfo()->getName(),
+                                 true));
+  }
+  virtual void MacroExpands(const Token &MacroNameTok, const MacroInfo* MI,
+                            SourceRange Range) {
+    Macros.push_back(MacroAction(MacroNameTok.getLocation(),
+                                 MacroNameTok.getIdentifierInfo()->getName(),
+                                 false));
+  }
+};
+
+}
+
+TEST_F(SourceManagerTest, isBeforeInTranslationUnitWithMacroInInclude) {
+  const char *header =
+    "#define MACRO_IN_INCLUDE 0\n";
+
+  const char *main =
+    "#define M(x) x\n"
+    "#define INC \"/test-header.h\"\n"
+    "#include M(INC)\n"
+    "#define INC2 </test-header.h>\n"
+    "#include M(INC2)\n";
+
+  MemoryBuffer *headerBuf = MemoryBuffer::getMemBuffer(header);
+  MemoryBuffer *mainBuf = MemoryBuffer::getMemBuffer(main);
+  SourceMgr.createMainFileIDForMemBuffer(mainBuf);
+
+  const FileEntry *headerFile = FileMgr.getVirtualFile("/test-header.h",
+                                                 headerBuf->getBufferSize(), 0);
+  SourceMgr.overrideFileContents(headerFile, headerBuf);
+
+  VoidModuleLoader ModLoader;
+  HeaderSearch HeaderInfo(FileMgr, Diags, LangOpts, &*Target);
+  Preprocessor PP(Diags, LangOpts,
+                  Target.getPtr(),
+                  SourceMgr, HeaderInfo, ModLoader,
+                  /*IILookup =*/ 0,
+                  /*OwnsHeaderSearch =*/false,
+                  /*DelayInitialization =*/ false);
+
+  std::vector<MacroAction> Macros;
+  PP.addPPCallbacks(new MacroTracker(Macros));
+
+  PP.EnterMainSourceFile();
+
+  std::vector<Token> toks;
+  while (1) {
+    Token tok;
+    PP.Lex(tok);
+    if (tok.is(tok::eof))
+      break;
+    toks.push_back(tok);
+  }
+
+  // Make sure we got the tokens that we expected.
+  ASSERT_EQ(0U, toks.size());
+
+  ASSERT_EQ(9U, Macros.size());
+  // #define M(x) x
+  ASSERT_TRUE(Macros[0].isDefinition);
+  ASSERT_EQ("M", Macros[0].Name);
+  // #define INC "/test-header.h"
+  ASSERT_TRUE(Macros[1].isDefinition);
+  ASSERT_EQ("INC", Macros[1].Name);
+  // M expansion in #include M(INC)
+  ASSERT_FALSE(Macros[2].isDefinition);
+  ASSERT_EQ("M", Macros[2].Name);
+  // INC expansion in #include M(INC)
+  ASSERT_FALSE(Macros[3].isDefinition);
+  ASSERT_EQ("INC", Macros[3].Name);
+  // #define MACRO_IN_INCLUDE 0
+  ASSERT_TRUE(Macros[4].isDefinition);
+  ASSERT_EQ("MACRO_IN_INCLUDE", Macros[4].Name);
+  // #define INC2 </test-header.h>
+  ASSERT_TRUE(Macros[5].isDefinition);
+  ASSERT_EQ("INC2", Macros[5].Name);
+  // M expansion in #include M(INC2)
+  ASSERT_FALSE(Macros[6].isDefinition);
+  ASSERT_EQ("M", Macros[6].Name);
+  // INC2 expansion in #include M(INC2)
+  ASSERT_FALSE(Macros[7].isDefinition);
+  ASSERT_EQ("INC2", Macros[7].Name);
+  // #define MACRO_IN_INCLUDE 0
+  ASSERT_TRUE(Macros[8].isDefinition);
+  ASSERT_EQ("MACRO_IN_INCLUDE", Macros[8].Name);
+
+  // The INC expansion in #include M(INC) comes before the first
+  // MACRO_IN_INCLUDE definition of the included file.
+  EXPECT_TRUE(SourceMgr.isBeforeInTranslationUnit(Macros[3].Loc, Macros[4].Loc));
+
+  // The INC2 expansion in #include M(INC2) comes before the second
+  // MACRO_IN_INCLUDE definition of the included file.
+  EXPECT_TRUE(SourceMgr.isBeforeInTranslationUnit(Macros[7].Loc, Macros[8].Loc));
+}
+
 #endif
 
 } // anonymous namespace