SpecialCaseList: Add support for parsing multiple input files.

Summary:
This change allows users to create SpecialCaseList objects from
multiple local files. This is needed to implement a proper support
for -fsanitize-blacklist flag (allow users to specify multiple blacklists,
in addition to default blacklist, see PR22431).

DFSan can also benefit from this change, as DFSan instrumentation pass now
accepts ABI-lists both from -fsanitize-blacklist= and -mllvm -dfsan-abilist flags.

Go bindings are fixed accordingly.

Test Plan: regression test suite

Reviewers: pcc

Subscribers: llvm-commits, axw, kcc

Differential Revision: http://reviews.llvm.org/D7367

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@228155 91177308-0d34-0410-b5e6-96231b3b80d8

bindings/go/llvm/InstrumentationBindings.cpp

@@ -37,6 +37,11 @@ void LLVMAddMemorySanitizerPass(LLVMPassManagerRef PM) {
 }
 
 void LLVMAddDataFlowSanitizerPass(LLVMPassManagerRef PM,
-                                  const char *ABIListFile) {
-  unwrap(PM)->add(createDataFlowSanitizerPass(ABIListFile));
+                                  int ABIListFilesNum,
+                                  const char **ABIListFiles) {
+  std::vector<std::string> ABIListFilesVec;
+  for (int i = 0; i != ABIListFilesNum; ++i) {
+    ABIListFilesVec.push_back(ABIListFiles[i]);
+  }
+  unwrap(PM)->add(createDataFlowSanitizerPass(ABIListFilesVec));
 }

bindings/go/llvm/InstrumentationBindings.h

@@ -28,8 +28,8 @@ void LLVMAddAddressSanitizerFunctionPass(LLVMPassManagerRef PM);
 void LLVMAddAddressSanitizerModulePass(LLVMPassManagerRef PM);
 void LLVMAddThreadSanitizerPass(LLVMPassManagerRef PM);
 void LLVMAddMemorySanitizerPass(LLVMPassManagerRef PM);
-void LLVMAddDataFlowSanitizerPass(LLVMPassManagerRef PM,
-                                  const char *ABIListFile);
+void LLVMAddDataFlowSanitizerPass(LLVMPassManagerRef PM, int ABIListFilesNum,
+                                  const char **ABIListFiles);
 
 #ifdef __cplusplus
 }

bindings/go/llvm/transforms_instrumentation.go

@@ -36,8 +36,11 @@ func (pm PassManager) AddMemorySanitizerPass() {
 	C.LLVMAddMemorySanitizerPass(pm.C)
 }
 
-func (pm PassManager) AddDataFlowSanitizerPass(abilist string) {
-	cabilist := C.CString(abilist)
-	defer C.free(unsafe.Pointer(cabilist))
-	C.LLVMAddDataFlowSanitizerPass(pm.C, cabilist)
+func (pm PassManager) AddDataFlowSanitizerPass(abilist []string) {
+	abiliststrs := make([]*C.char, len(abilist))
+	for i, arg := range abilist {
+		abiliststrs[i] = C.CString(arg)
+		defer C.free(unsafe.Pointer(abiliststrs[i]))
+	}
+	C.LLVMAddDataFlowSanitizerPass(pm.C, C.int(len(abilist)), &abiliststrs[0])
 }

include/llvm/Support/SpecialCaseList.h

@@ -49,6 +49,8 @@
 #define LLVM_SUPPORT_SPECIALCASELIST_H
 
 #include "llvm/ADT/StringMap.h"
+#include <string>
+#include <vector>
 
 namespace llvm {
 class MemoryBuffer;
@@ -57,18 +59,18 @@ class StringRef;
 
 class SpecialCaseList {
 public:
-  /// Parses the special case list from a file. If Path is empty, returns
-  /// an empty special case list. On failure, returns 0 and writes an error
-  /// message to string.
-  static std::unique_ptr<SpecialCaseList> create(StringRef Path,
-                                                  std::string &Error);
+  /// Parses the special case list entries from files. On failure, returns
+  /// 0 and writes an error message to string.
+  static std::unique_ptr<SpecialCaseList>
+  create(const std::vector<std::string> &Paths, std::string &Error);
   /// Parses the special case list from a memory buffer. On failure, returns
   /// 0 and writes an error message to string.
   static std::unique_ptr<SpecialCaseList> create(const MemoryBuffer *MB,
-                                                  std::string &Error);
-  /// Parses the special case list from a file. On failure, reports a fatal
-  /// error.
-  static std::unique_ptr<SpecialCaseList> createOrDie(StringRef Path);
+                                                 std::string &Error);
+  /// Parses the special case list entries from files. On failure, reports a
+  /// fatal error.
+  static std::unique_ptr<SpecialCaseList>
+  createOrDie(const std::vector<std::string> &Paths);
 
   ~SpecialCaseList();
 
@@ -85,11 +87,15 @@ private:
   SpecialCaseList &operator=(SpecialCaseList const &) LLVM_DELETED_FUNCTION;
 
   struct Entry;
-  StringMap<StringMap<Entry> > Entries;
+  StringMap<StringMap<Entry>> Entries;
+  StringMap<StringMap<std::string>> Regexps;
+  bool IsCompiled;
 
   SpecialCaseList();
   /// Parses just-constructed SpecialCaseList entries from a memory buffer.
   bool parse(const MemoryBuffer *MB, std::string &Error);
+  /// compile() should be called once, after parsing all the memory buffers.
+  void compile();
 };
 
 }  // namespace llvm

include/llvm/Transforms/Instrumentation.h

@@ -86,17 +86,17 @@ FunctionPass *createMemorySanitizerPass(int TrackOrigins = 0);
 FunctionPass *createThreadSanitizerPass();
 
 // Insert DataFlowSanitizer (dynamic data flow analysis) instrumentation
-ModulePass *createDataFlowSanitizerPass(StringRef ABIListFile = StringRef(),
-                                        void *(*getArgTLS)() = nullptr,
-                                        void *(*getRetValTLS)() = nullptr);
+ModulePass *createDataFlowSanitizerPass(
+    const std::vector<std::string> &ABIListFiles = std::vector<std::string>(),
+    void *(*getArgTLS)() = nullptr, void *(*getRetValTLS)() = nullptr);
 
 // Insert SanitizerCoverage instrumentation.
 ModulePass *createSanitizerCoverageModulePass(int CoverageLevel);
 
 #if defined(__GNUC__) && defined(__linux__) && !defined(ANDROID)
-inline ModulePass *createDataFlowSanitizerPassForJIT(StringRef ABIListFile =
-                                                         StringRef()) {
-  return createDataFlowSanitizerPass(ABIListFile, getDFSanArgTLSPtrForJIT,
+inline ModulePass *createDataFlowSanitizerPassForJIT(
+    const std::vector<std::string> &ABIListFiles = std::vector<std::string>()) {
+  return createDataFlowSanitizerPass(ABIListFiles, getDFSanArgTLSPtrForJIT,
                                      getDFSanRetValTLSPtrForJIT);
 }
 #endif

lib/Support/SpecialCaseList.cpp

@@ -46,19 +46,27 @@ struct SpecialCaseList::Entry {
   }
 };
 
-SpecialCaseList::SpecialCaseList() : Entries() {}
+SpecialCaseList::SpecialCaseList() : Entries(), Regexps(), IsCompiled(false) {}
 
-std::unique_ptr<SpecialCaseList> SpecialCaseList::create(StringRef Path,
-                                                         std::string &Error) {
-  if (Path.empty())
-    return std::unique_ptr<SpecialCaseList>(new SpecialCaseList());
-  ErrorOr<std::unique_ptr<MemoryBuffer>> FileOrErr =
-      MemoryBuffer::getFile(Path);
-  if (std::error_code EC = FileOrErr.getError()) {
-    Error = (Twine("Can't open file '") + Path + "': " + EC.message()).str();
-    return nullptr;
+std::unique_ptr<SpecialCaseList>
+SpecialCaseList::create(const std::vector<std::string> &Paths,
+                        std::string &Error) {
+  std::unique_ptr<SpecialCaseList> SCL(new SpecialCaseList());
+  for (auto Path : Paths) {
+    ErrorOr<std::unique_ptr<MemoryBuffer>> FileOrErr =
+        MemoryBuffer::getFile(Path);
+    if (std::error_code EC = FileOrErr.getError()) {
+      Error = (Twine("can't open file '") + Path + "': " + EC.message()).str();
+      return nullptr;
+    }
+    std::string ParseError;
+    if (!SCL->parse(FileOrErr.get().get(), ParseError)) {
+      Error = (Twine("error parsing file '") + Path + "': " + ParseError).str();
+      return nullptr;
+    }
   }
-  return create(FileOrErr.get().get(), Error);
+  SCL->compile();
+  return SCL;
 }
 
 std::unique_ptr<SpecialCaseList> SpecialCaseList::create(const MemoryBuffer *MB,
@@ -66,12 +74,14 @@ std::unique_ptr<SpecialCaseList> SpecialCaseList::create(const MemoryBuffer *MB,
   std::unique_ptr<SpecialCaseList> SCL(new SpecialCaseList());
   if (!SCL->parse(MB, Error))
     return nullptr;
+  SCL->compile();
   return SCL;
 }
 
-std::unique_ptr<SpecialCaseList> SpecialCaseList::createOrDie(StringRef Path) {
+std::unique_ptr<SpecialCaseList>
+SpecialCaseList::createOrDie(const std::vector<std::string> &Paths) {
   std::string Error;
-  if (auto SCL = create(Path, Error))
+  if (auto SCL = create(Paths, Error))
     return SCL;
   report_fatal_error(Error);
 }
@@ -80,12 +90,8 @@ bool SpecialCaseList::parse(const MemoryBuffer *MB, std::string &Error) {
   // Iterate through each line in the blacklist file.
   SmallVector<StringRef, 16> Lines;
   SplitString(MB->getBuffer(), Lines, "\n\r");
-  StringMap<StringMap<std::string> > Regexps;
-  assert(Entries.empty() &&
-         "parse() should be called on an empty SpecialCaseList");
   int LineNo = 1;
-  for (SmallVectorImpl<StringRef>::iterator I = Lines.begin(), E = Lines.end();
-       I != E; ++I, ++LineNo) {
+  for (auto I = Lines.begin(), E = Lines.end(); I != E; ++I, ++LineNo) {
     // Ignore empty lines and lines starting with "#"
     if (I->empty() || I->startswith("#"))
       continue;
@@ -94,7 +100,7 @@ bool SpecialCaseList::parse(const MemoryBuffer *MB, std::string &Error) {
     StringRef Prefix = SplitLine.first;
     if (SplitLine.second.empty()) {
       // Missing ':' in the line.
-      Error = (Twine("Malformed line ") + Twine(LineNo) + ": '" +
+      Error = (Twine("malformed line ") + Twine(LineNo) + ": '" +
                SplitLine.first + "'").str();
       return false;
     }
@@ -119,7 +125,7 @@ bool SpecialCaseList::parse(const MemoryBuffer *MB, std::string &Error) {
     Regex CheckRE(Regexp);
     std::string REError;
     if (!CheckRE.isValid(REError)) {
-      Error = (Twine("Malformed regex in line ") + Twine(LineNo) + ": '" +
+      Error = (Twine("malformed regex in line ") + Twine(LineNo) + ": '" +
                SplitLine.second + "': " + REError).str();
       return false;
     }
@@ -129,10 +135,14 @@ bool SpecialCaseList::parse(const MemoryBuffer *MB, std::string &Error) {
       Regexps[Prefix][Category] += "|";
     Regexps[Prefix][Category] += "^" + Regexp + "$";
   }
+  return true;
+}
 
+void SpecialCaseList::compile() {
+  assert(!IsCompiled && "compile() should only be called once");
   // Iterate through each of the prefixes, and create Regexs for them.
-  for (StringMap<StringMap<std::string> >::const_iterator I = Regexps.begin(),
-                                                          E = Regexps.end();
+  for (StringMap<StringMap<std::string>>::const_iterator I = Regexps.begin(),
+                                                         E = Regexps.end();
        I != E; ++I) {
     for (StringMap<std::string>::const_iterator II = I->second.begin(),
                                                 IE = I->second.end();
@@ -140,13 +150,15 @@ bool SpecialCaseList::parse(const MemoryBuffer *MB, std::string &Error) {
       Entries[I->getKey()][II->getKey()].RegEx.reset(new Regex(II->getValue()));
     }
   }
-  return true;
+  Regexps.clear();
+  IsCompiled = true;
 }
 
 SpecialCaseList::~SpecialCaseList() {}
 
 bool SpecialCaseList::inSection(StringRef Section, StringRef Query,
                                 StringRef Category) const {
+  assert(IsCompiled && "SpecialCaseList::compile() was not called!");
   StringMap<StringMap<Entry> >::const_iterator I = Entries.find(Section);
   if (I == Entries.end()) return false;
   StringMap<Entry>::const_iterator II = I->second.find(Category);

lib/Transforms/Instrumentation/DataFlowSanitizer.cpp

@@ -83,14 +83,14 @@ static cl::opt<bool> ClPreserveAlignment(
     cl::desc("respect alignment requirements provided by input IR"), cl::Hidden,
     cl::init(false));
 
-// The ABI list file controls how shadow parameters are passed.  The pass treats
+// The ABI list files control how shadow parameters are passed. The pass treats
 // every function labelled "uninstrumented" in the ABI list file as conforming
 // to the "native" (i.e. unsanitized) ABI.  Unless the ABI list contains
 // additional annotations for those functions, a call to one of those functions
 // will produce a warning message, as the labelling behaviour of the function is
 // unknown.  The other supported annotations are "functional" and "discard",
 // which are described below under DataFlowSanitizer::WrapperKind.
-static cl::opt<std::string> ClABIListFile(
+static cl::list<std::string> ClABIListFiles(
     "dfsan-abilist",
     cl::desc("File listing native ABI functions and how the pass treats them"),
     cl::Hidden);
@@ -141,7 +141,9 @@ class DFSanABIList {
   std::unique_ptr<SpecialCaseList> SCL;
 
  public:
-  DFSanABIList(std::unique_ptr<SpecialCaseList> SCL) : SCL(std::move(SCL)) {}
+  DFSanABIList() {}
+
+  void set(std::unique_ptr<SpecialCaseList> List) { SCL = std::move(List); }
 
   /// Returns whether either this function or its source file are listed in the
   /// given category.
@@ -264,9 +266,9 @@ class DataFlowSanitizer : public ModulePass {
   Constant *getOrBuildTrampolineFunction(FunctionType *FT, StringRef FName);
 
  public:
-  DataFlowSanitizer(StringRef ABIListFile = StringRef(),
-                    void *(*getArgTLS)() = nullptr,
-                    void *(*getRetValTLS)() = nullptr);
+  DataFlowSanitizer(
+      const std::vector<std::string> &ABIListFiles = std::vector<std::string>(),
+      void *(*getArgTLS)() = nullptr, void *(*getRetValTLS)() = nullptr);
   static char ID;
   bool doInitialization(Module &M) override;
   bool runOnModule(Module &M) override;
@@ -351,18 +353,21 @@ char DataFlowSanitizer::ID;
 INITIALIZE_PASS(DataFlowSanitizer, "dfsan",
                 "DataFlowSanitizer: dynamic data flow analysis.", false, false)
 
-ModulePass *llvm::createDataFlowSanitizerPass(StringRef ABIListFile,
-                                              void *(*getArgTLS)(),
-                                              void *(*getRetValTLS)()) {
-  return new DataFlowSanitizer(ABIListFile, getArgTLS, getRetValTLS);
+ModulePass *
+llvm::createDataFlowSanitizerPass(const std::vector<std::string> &ABIListFiles,
+                                  void *(*getArgTLS)(),
+                                  void *(*getRetValTLS)()) {
+  return new DataFlowSanitizer(ABIListFiles, getArgTLS, getRetValTLS);
 }
 
-DataFlowSanitizer::DataFlowSanitizer(StringRef ABIListFile,
-                                     void *(*getArgTLS)(),
-                                     void *(*getRetValTLS)())
-    : ModulePass(ID), GetArgTLSPtr(getArgTLS), GetRetvalTLSPtr(getRetValTLS),
-      ABIList(SpecialCaseList::createOrDie(ABIListFile.empty() ? ClABIListFile
-                                                               : ABIListFile)) {
+DataFlowSanitizer::DataFlowSanitizer(
+    const std::vector<std::string> &ABIListFiles, void *(*getArgTLS)(),
+    void *(*getRetValTLS)())
+    : ModulePass(ID), GetArgTLSPtr(getArgTLS), GetRetvalTLSPtr(getRetValTLS) {
+  std::vector<std::string> AllABIListFiles(std::move(ABIListFiles));
+  AllABIListFiles.insert(AllABIListFiles.end(), ClABIListFiles.begin(),
+                         ClABIListFiles.end());
+  ABIList.set(SpecialCaseList::createOrDie(AllABIListFiles));
 }
 
 FunctionType *DataFlowSanitizer::getArgsFunctionType(FunctionType *T) {

unittests/Support/SpecialCaseListTest.cpp

@@ -7,6 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/SpecialCaseList.h"
 #include "gtest/gtest.h"
@@ -30,6 +31,16 @@ protected:
     assert(Error == "");
     return SCL;
   }
+
+  std::string makeSpecialCaseListFile(StringRef Contents) {
+    int FD;
+    SmallString<64> Path;
+    sys::fs::createTemporaryFile("SpecialCaseListTest", "temp", FD, Path);
+    raw_fd_ostream OF(FD, true, true);
+    OF << Contents;
+    OF.close();
+    return Path.str();
+  }
 };
 
 TEST_F(SpecialCaseListTest, Basic) {
@@ -86,17 +97,18 @@ TEST_F(SpecialCaseListTest, Substring) {
 TEST_F(SpecialCaseListTest, InvalidSpecialCaseList) {
   std::string Error;
   EXPECT_EQ(nullptr, makeSpecialCaseList("badline", Error));
-  EXPECT_EQ("Malformed line 1: 'badline'", Error);
+  EXPECT_EQ("malformed line 1: 'badline'", Error);
   EXPECT_EQ(nullptr, makeSpecialCaseList("src:bad[a-", Error));
-  EXPECT_EQ("Malformed regex in line 1: 'bad[a-': invalid character range",
+  EXPECT_EQ("malformed regex in line 1: 'bad[a-': invalid character range",
             Error);
   EXPECT_EQ(nullptr, makeSpecialCaseList("src:a.c\n"
                                    "fun:fun(a\n",
                                    Error));
-  EXPECT_EQ("Malformed regex in line 2: 'fun(a': parentheses not balanced",
+  EXPECT_EQ("malformed regex in line 2: 'fun(a': parentheses not balanced",
             Error);
-  EXPECT_EQ(nullptr, SpecialCaseList::create("unexisting", Error));
-  EXPECT_EQ(0U, Error.find("Can't open file 'unexisting':"));
+  std::vector<std::string> Files(1, "unexisting");
+  EXPECT_EQ(nullptr, SpecialCaseList::create(Files, Error));
+  EXPECT_EQ(0U, Error.find("can't open file 'unexisting':"));
 }
 
 TEST_F(SpecialCaseListTest, EmptySpecialCaseList) {
@@ -104,6 +116,20 @@ TEST_F(SpecialCaseListTest, EmptySpecialCaseList) {
   EXPECT_FALSE(SCL->inSection("foo", "bar"));
 }
 
+TEST_F(SpecialCaseListTest, MultipleBlacklists) {
+  std::vector<std::string> Files;
+  Files.push_back(makeSpecialCaseListFile("src:bar\n"
+                                          "src:*foo*\n"
+                                          "src:ban=init\n"));
+  Files.push_back(makeSpecialCaseListFile("src:baz\n"
+                                          "src:*fog*\n"));
+  auto SCL = SpecialCaseList::createOrDie(Files);
+  EXPECT_TRUE(SCL->inSection("src", "bar"));
+  EXPECT_TRUE(SCL->inSection("src", "baz"));
+  EXPECT_FALSE(SCL->inSection("src", "ban"));
+  EXPECT_TRUE(SCL->inSection("src", "ban", "init"));
+  EXPECT_TRUE(SCL->inSection("src", "tomfoolery"));
+  EXPECT_TRUE(SCL->inSection("src", "tomfoglery"));
 }
 
-
+}