Refactor DynamicLibrary so searching for a symbol will have a defined order and

libraries are properly unloaded when llvm_shutdown is called.

Summary:
This was mostly affecting usage of the JIT, where storing the library handles in
a set made iteration unordered/undefined. This lead to disagreement between the
JIT and native code as to what the address and implementation of particularly on
Windows with stdlib functions:

JIT: putenv_s("TEST", "VALUE") // called msvcrt.dll, putenv_s
JIT: getenv("TEST") -> "VALUE" // called msvcrt.dll, getenv
Native: getenv("TEST") -> NULL // called ucrt.dll, getenv

Also fixed is the issue of DynamicLibrary::getPermanentLibrary(0,0) on Windows
not giving priority to the process' symbols as it did on Unix.

Reviewers: chapuni, v.g.vassilev, lhames

Reviewed By: lhames

Subscribers: danalbert, srhines, mgorny, vsk, llvm-commits

Differential Revision: https://reviews.llvm.org/D30107

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@301153 91177308-0d34-0410-b5e6-96231b3b80d8

include/llvm/Support/DynamicLibrary.h

@@ -58,7 +58,7 @@ namespace sys {
     void *getAddressOfSymbol(const char *symbolName);
 
     /// This function permanently loads the dynamic library at the given path.
-    /// The library will only be unloaded when the program terminates.
+    /// The library will only be unloaded when llvm_shutdown() is called.
     /// This returns a valid DynamicLibrary instance on success and an invalid
     /// instance on failure (see isValid()). \p *errMsg will only be modified
     /// if the library fails to load.
@@ -71,7 +71,8 @@ namespace sys {
     /// Registers an externally loaded library. The library will be unloaded
     /// when the program terminates.
     ///
-    /// It is safe to call this function multiple times for the same library.
+    /// It is safe to call this function multiple times for the same library,
+    /// though ownership is only taken if there was no error.
     ///
     /// \returns An empty \p DynamicLibrary if the library was already loaded.
     static DynamicLibrary addPermanentLibrary(void *handle,
@@ -106,6 +107,8 @@ namespace sys {
     /// libraries.
     /// @brief Add searchable symbol/value pair.
     static void AddSymbol(StringRef symbolName, void *symbolValue);
+
+    class HandleSet;
   };
 
 } // End sys namespace

lib/Support/CMakeLists.txt

@@ -130,7 +130,6 @@ add_llvm_library(LLVMSupport
   Process.cpp
   Program.cpp
   RWMutex.cpp
-  SearchForAddressOfSpecialSymbol.cpp
   Signals.cpp
   TargetRegistry.cpp
   ThreadLocal.cpp

lib/Support/DynamicLibrary.cpp

@@ -20,169 +20,164 @@
 #include "llvm/Support/Mutex.h"
 #include <cstdio>
 #include <cstring>
+#include <vector>
 
-// Collection of symbol name/value pairs to be searched prior to any libraries.
-static llvm::ManagedStatic<llvm::StringMap<void *> > ExplicitSymbols;
-static llvm::ManagedStatic<llvm::sys::SmartMutex<true> > SymbolsMutex;
-
-void llvm::sys::DynamicLibrary::AddSymbol(StringRef symbolName,
-                                          void *symbolValue) {
-  SmartScopedLock<true> lock(*SymbolsMutex);
-  (*ExplicitSymbols)[symbolName] = symbolValue;
-}
-
-char llvm::sys::DynamicLibrary::Invalid = 0;
-
-#ifdef LLVM_ON_WIN32
-
-#include "Windows/DynamicLibrary.inc"
-
-#else
-
-#if defined(HAVE_DLFCN_H) && defined(HAVE_DLOPEN)
-#include <dlfcn.h>
 using namespace llvm;
 using namespace llvm::sys;
 
-//===----------------------------------------------------------------------===//
-//=== WARNING: Implementation here must contain only TRULY operating system
-//===          independent code.
-//===----------------------------------------------------------------------===//
+// All methods for HandleSet should be used holding SymbolsMutex.
+class DynamicLibrary::HandleSet {
+  typedef std::vector<void *> HandleList;
+  HandleList Handles;
+  void *Process;
 
-static llvm::ManagedStatic<DenseSet<void *> > OpenedHandles;
+public:
+  static void *DLOpen(const char *Filename, std::string *Err);
+  static void DLClose(void *Handle);
+  static void *DLSym(void *Handle, const char *Symbol);
 
-DynamicLibrary DynamicLibrary::getPermanentLibrary(const char *filename,
-                                                   std::string *errMsg) {
-  SmartScopedLock<true> lock(*SymbolsMutex);
+  HandleSet() : Process(nullptr) {}
+  ~HandleSet();
 
-  void *handle = dlopen(filename, RTLD_LAZY|RTLD_GLOBAL);
-  if (!handle) {
-    if (errMsg) *errMsg = dlerror();
-    return DynamicLibrary();
+  HandleList::iterator Find(void *Handle) {
+    return std::find(Handles.begin(), Handles.end(), Handle);
   }
 
-#ifdef __CYGWIN__
-  // Cygwin searches symbols only in the main
-  // with the handle of dlopen(NULL, RTLD_GLOBAL).
-  if (!filename)
-    handle = RTLD_DEFAULT;
-#endif
+  bool Contains(void *Handle) {
+    return Handle == Process || Find(Handle) != Handles.end();
+  }
 
-  // If we've already loaded this library, dlclose() the handle in order to
-  // keep the internal refcount at +1.
-  if (!OpenedHandles->insert(handle).second)
-    dlclose(handle);
+  bool AddLibrary(void *Handle, bool IsProcess = false, bool CanClose = true) {
+#ifdef LLVM_ON_WIN32
+    assert((Handle == this ? IsProcess : !IsProcess) && "Bad Handle.");
+#endif
 
-  return DynamicLibrary(handle);
-}
+    if (LLVM_LIKELY(!IsProcess)) {
+      if (Find(Handle) != Handles.end()) {
+        if (CanClose)
+          DLClose(Handle);
+        return false;
+      }
+      Handles.push_back(Handle);
+    } else {
+#ifndef LLVM_ON_WIN32
+      if (Process) {
+        if (CanClose)
+          DLClose(Process);
+        if (Process == Handle)
+          return false;
+      }
+#endif
+      Process = Handle;
+    }
+    return true;
+  }
 
-DynamicLibrary DynamicLibrary::addPermanentLibrary(void *handle,
-                                                   std::string *errMsg) {
-  SmartScopedLock<true> lock(*SymbolsMutex);
-  // If we've already loaded this library, tell the caller.
-  if (!OpenedHandles->insert(handle).second) {
-    if (errMsg) *errMsg = "Library already loaded";
-    return DynamicLibrary();
+  void *Lookup(const char *Symbol) {
+    // Process handle gets first try.
+    if (Process) {
+      if (void *Ptr = DLSym(Process, Symbol))
+        return Ptr;
+#ifndef NDEBUG
+      for (void *Handle : Handles)
+        assert(!DLSym(Handle, Symbol) && "Symbol exists in non process handle");
+#endif
+    } else {
+      // Iterate in reverse, so newer libraries/symbols override older.
+      for (auto &&I = Handles.rbegin(), E = Handles.rend(); I != E; ++I) {
+        if (void *Ptr = DLSym(*I, Symbol))
+          return Ptr;
+      }
+    }
+    return nullptr;
   }
+};
 
-  return DynamicLibrary(handle);
+namespace {
+// Collection of symbol name/value pairs to be searched prior to any libraries.
+static llvm::ManagedStatic<llvm::StringMap<void *>> ExplicitSymbols;
+// Collection of known library handles.
+static llvm::ManagedStatic<DynamicLibrary::HandleSet> OpenedHandles;
+// Lock for ExplicitSymbols and OpenedHandles.
+static llvm::ManagedStatic<llvm::sys::SmartMutex<true>> SymbolsMutex;
 }
 
-void *DynamicLibrary::getAddressOfSymbol(const char *symbolName) {
-  if (!isValid())
-    return nullptr;
-  return dlsym(Data, symbolName);
-}
+#ifdef LLVM_ON_WIN32
+
+#include "Windows/DynamicLibrary.inc"
 
 #else
 
-using namespace llvm;
-using namespace llvm::sys;
+#include "Unix/DynamicLibrary.inc"
+
+#endif
 
-DynamicLibrary DynamicLibrary::getPermanentLibrary(const char *filename,
-                                                   std::string *errMsg) {
-  if (errMsg) *errMsg = "dlopen() not supported on this platform";
-  return DynamicLibrary();
+char DynamicLibrary::Invalid;
+
+namespace llvm {
+void *SearchForAddressOfSpecialSymbol(const char *SymbolName) {
+  return DoSearch(SymbolName); // DynamicLibrary.inc
+}
 }
 
-void *DynamicLibrary::getAddressOfSymbol(const char *symbolName) {
-  return NULL;
+void DynamicLibrary::AddSymbol(StringRef SymbolName, void *SymbolValue) {
+  SmartScopedLock<true> Lock(*SymbolsMutex);
+  (*ExplicitSymbols)[SymbolName] = SymbolValue;
 }
 
-#endif
+DynamicLibrary DynamicLibrary::getPermanentLibrary(const char *FileName,
+                                                   std::string *Err) {
+  SmartScopedLock<true> Lock(*SymbolsMutex);
+  void *Handle = HandleSet::DLOpen(FileName, Err);
+  if (Handle != &Invalid)
+    OpenedHandles->AddLibrary(Handle, /*IsProcess*/ FileName == nullptr);
 
-namespace llvm {
-void *SearchForAddressOfSpecialSymbol(const char* symbolName);
+  return DynamicLibrary(Handle);
 }
 
-void* DynamicLibrary::SearchForAddressOfSymbol(const char *symbolName) {
+DynamicLibrary DynamicLibrary::addPermanentLibrary(void *Handle,
+                                                   std::string *Err) {
   SmartScopedLock<true> Lock(*SymbolsMutex);
+  // If we've already loaded this library, tell the caller.
+  if (!OpenedHandles->AddLibrary(Handle, /*IsProcess*/false, /*CanClose*/false))
+    *Err = "Library already loaded";
 
-  // First check symbols added via AddSymbol().
-  if (ExplicitSymbols.isConstructed()) {
-    StringMap<void *>::iterator i = ExplicitSymbols->find(symbolName);
+  return DynamicLibrary(Handle);
+}
 
-    if (i != ExplicitSymbols->end())
-      return i->second;
-  }
+void *DynamicLibrary::getAddressOfSymbol(const char *SymbolName) {
+  if (!isValid())
+    return nullptr;
+  return HandleSet::DLSym(Data, SymbolName);
+}
 
-#if defined(HAVE_DLFCN_H) && defined(HAVE_DLOPEN)
-  // Now search the libraries.
-  if (OpenedHandles.isConstructed()) {
-    for (DenseSet<void *>::iterator I = OpenedHandles->begin(),
-         E = OpenedHandles->end(); I != E; ++I) {
-      //lt_ptr ptr = lt_dlsym(*I, symbolName);
-      void *ptr = dlsym(*I, symbolName);
-      if (ptr) {
-        return ptr;
-      }
-    }
-  }
-#endif
+void *DynamicLibrary::SearchForAddressOfSymbol(const char *SymbolName) {
+  {
+    SmartScopedLock<true> Lock(*SymbolsMutex);
 
-  if (void *Result = llvm::SearchForAddressOfSpecialSymbol(symbolName))
-    return Result;
+    // First check symbols added via AddSymbol().
+    if (ExplicitSymbols.isConstructed()) {
+      StringMap<void *>::iterator i = ExplicitSymbols->find(SymbolName);
 
-// This macro returns the address of a well-known, explicit symbol
-#define EXPLICIT_SYMBOL(SYM) \
-   if (!strcmp(symbolName, #SYM)) return &SYM
+      if (i != ExplicitSymbols->end())
+        return i->second;
+    }
 
-// On linux we have a weird situation. The stderr/out/in symbols are both
-// macros and global variables because of standards requirements. So, we
-// boldly use the EXPLICIT_SYMBOL macro without checking for a #define first.
-#if defined(__linux__) and !defined(__ANDROID__)
-  {
-    EXPLICIT_SYMBOL(stderr);
-    EXPLICIT_SYMBOL(stdout);
-    EXPLICIT_SYMBOL(stdin);
-  }
-#else
-  // For everything else, we want to check to make sure the symbol isn't defined
-  // as a macro before using EXPLICIT_SYMBOL.
-  {
-#ifndef stdin
-    EXPLICIT_SYMBOL(stdin);
-#endif
-#ifndef stdout
-    EXPLICIT_SYMBOL(stdout);
-#endif
-#ifndef stderr
-    EXPLICIT_SYMBOL(stderr);
-#endif
+    // Now search the libraries.
+    if (OpenedHandles.isConstructed()) {
+      if (void *Ptr = OpenedHandles->Lookup(SymbolName))
+        return Ptr;
+    }
   }
-#endif
-#undef EXPLICIT_SYMBOL
 
-  return nullptr;
+  return llvm::SearchForAddressOfSpecialSymbol(SymbolName);
 }
 
-#endif // LLVM_ON_WIN32
-
 //===----------------------------------------------------------------------===//
 // C API.
 //===----------------------------------------------------------------------===//
 
-LLVMBool LLVMLoadLibraryPermanently(const char* Filename) {
+LLVMBool LLVMLoadLibraryPermanently(const char *Filename) {
   return llvm::sys::DynamicLibrary::LoadLibraryPermanently(Filename);
 }
 

lib/Support/SearchForAddressOfSpecialSymbol.cpp

@@ -1,58 +0,0 @@
-//===- SearchForAddressOfSpecialSymbol.cpp - Function addresses -*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file pulls the addresses of certain symbols out of the linker.  It must
-//  include as few header files as possible because it declares the symbols as
-//  void*, which would conflict with the actual symbol type if any header
-//  declared it.
-//
-//===----------------------------------------------------------------------===//
-
-#include <string.h>
-
-// Must declare the symbols in the global namespace.
-static void *DoSearch(const char* symbolName) {
-#define EXPLICIT_SYMBOL(SYM) \
-   extern void *SYM; if (!strcmp(symbolName, #SYM)) return &SYM
-
-  // If this is darwin, it has some funky issues, try to solve them here.  Some
-  // important symbols are marked 'private external' which doesn't allow
-  // SearchForAddressOfSymbol to find them.  As such, we special case them here,
-  // there is only a small handful of them.
-
-#ifdef __APPLE__
-  {
-    // __eprintf is sometimes used for assert() handling on x86.
-    //
-    // FIXME: Currently disabled when using Clang, as we don't always have our
-    // runtime support libraries available.
-#ifndef __clang__
-#ifdef __i386__
-    EXPLICIT_SYMBOL(__eprintf);
-#endif
-#endif
-  }
-#endif
-
-#ifdef __CYGWIN__
-  {
-    EXPLICIT_SYMBOL(_alloca);
-    EXPLICIT_SYMBOL(__main);
-  }
-#endif
-
-#undef EXPLICIT_SYMBOL
-  return nullptr;
-}
-
-namespace llvm {
-void *SearchForAddressOfSpecialSymbol(const char* symbolName) {
-  return DoSearch(symbolName);
-}
-}  // namespace llvm

lib/Support/Unix/DynamicLibrary.inc

@@ -0,0 +1,131 @@
+//===- Unix/DynamicLibrary.cpp - Unix DL Implementation ---------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file provides the UNIX specific implementation of DynamicLibrary.
+//
+//===----------------------------------------------------------------------===//
+
+#if defined(HAVE_DLFCN_H) && defined(HAVE_DLOPEN)
+#include <dlfcn.h>
+
+DynamicLibrary::HandleSet::~HandleSet() {
+  for (void *Handle : Handles)
+    ::dlclose(Handle);
+  if (Process)
+    ::dlclose(Process);
+}
+
+void *DynamicLibrary::HandleSet::DLOpen(const char *File, std::string *Err) {
+  void *Handle = ::dlopen(File, RTLD_LAZY|RTLD_GLOBAL);
+  if (!Handle) {
+    if (Err) *Err = ::dlerror();
+    return &DynamicLibrary::Invalid;
+  }
+
+#ifdef __CYGWIN__
+  // Cygwin searches symbols only in the main
+  // with the handle of dlopen(NULL, RTLD_GLOBAL).
+  if (!Filename)
+    Handle = RTLD_DEFAULT;
+#endif
+
+  return Handle;
+}
+
+void DynamicLibrary::HandleSet::DLClose(void *Handle) {
+  ::dlclose(Handle);
+}
+
+void *DynamicLibrary::HandleSet::DLSym(void *Handle, const char *Symbol) {
+  return ::dlsym(Handle, Symbol);
+}
+
+#else // !HAVE_DLOPEN
+
+DynamicLibrary::HandleSet::~HandleSet() {}
+
+void *DynamicLibrary::HandleSet::DLOpen(const char *File, std::string *Err) {
+  if (Err) *Err = "dlopen() not supported on this platform";
+  return &Invalid;
+}
+
+void DynamicLibrary::HandleSet::DLClose(void *Handle) {
+}
+
+void *DynamicLibrary::HandleSet::DLSym(void *Handle, const char *Symbol) {
+  return nullptr;
+}
+
+#endif
+
+// Must declare the symbols in the global namespace.
+static void *DoSearch(const char* SymbolName) {
+#define EXPLICIT_SYMBOL(SYM) \
+   extern void *SYM; if (!strcmp(SymbolName, #SYM)) return &SYM
+
+  // If this is darwin, it has some funky issues, try to solve them here.  Some
+  // important symbols are marked 'private external' which doesn't allow
+  // SearchForAddressOfSymbol to find them.  As such, we special case them here,
+  // there is only a small handful of them.
+
+#ifdef __APPLE__
+  {
+    // __eprintf is sometimes used for assert() handling on x86.
+    //
+    // FIXME: Currently disabled when using Clang, as we don't always have our
+    // runtime support libraries available.
+#ifndef __clang__
+#ifdef __i386__
+    EXPLICIT_SYMBOL(__eprintf);
+#endif
+#endif
+  }
+#endif
+
+#ifdef __CYGWIN__
+  {
+    EXPLICIT_SYMBOL(_alloca);
+    EXPLICIT_SYMBOL(__main);
+  }
+#endif
+
+#undef EXPLICIT_SYMBOL
+
+// This macro returns the address of a well-known, explicit symbol
+#define EXPLICIT_SYMBOL(SYM) \
+   if (!strcmp(SymbolName, #SYM)) return &SYM
+
+// On linux we have a weird situation. The stderr/out/in symbols are both
+// macros and global variables because of standards requirements. So, we
+// boldly use the EXPLICIT_SYMBOL macro without checking for a #define first.
+#if defined(__linux__) and !defined(__ANDROID__)
+  {
+    EXPLICIT_SYMBOL(stderr);
+    EXPLICIT_SYMBOL(stdout);
+    EXPLICIT_SYMBOL(stdin);
+  }
+#else
+  // For everything else, we want to check to make sure the symbol isn't defined
+  // as a macro before using EXPLICIT_SYMBOL.
+  {
+#ifndef stdin
+    EXPLICIT_SYMBOL(stdin);
+#endif
+#ifndef stdout
+    EXPLICIT_SYMBOL(stdout);
+#endif
+#ifndef stderr
+    EXPLICIT_SYMBOL(stderr);
+#endif
+  }
+#endif
+#undef EXPLICIT_SYMBOL
+
+  return nullptr;
+}

lib/Support/Windows/DynamicLibrary.inc

@@ -16,94 +16,141 @@
 #ifdef __MINGW32__
  #include <imagehlp.h>
 #else
- #include <dbghelp.h>
+ #include <Psapi.h>
 #endif
 
 #ifdef _MSC_VER
  #include <ntverp.h>
 #endif
 
-namespace llvm {
-
 //===----------------------------------------------------------------------===//
 //=== WARNING: Implementation here must contain only Win32 specific code
 //===          and must not be UNIX code.
 //===----------------------------------------------------------------------===//
 
-typedef BOOL (WINAPI *fpEnumerateLoadedModules)(HANDLE,PENUMLOADED_MODULES_CALLBACK64,PVOID);
-static fpEnumerateLoadedModules fEnumerateLoadedModules;
-static llvm::ManagedStatic<DenseSet<HMODULE> > OpenedHandles;
 
-static bool loadDebugHelp(void) {
-  HMODULE hLib = ::LoadLibraryW(L"Dbghelp.dll");
-  if (hLib) {
-    fEnumerateLoadedModules = (fpEnumerateLoadedModules)
-      ::GetProcAddress(hLib, "EnumerateLoadedModules64");
-  }
-  return fEnumerateLoadedModules != 0;
-}
+DynamicLibrary::HandleSet::~HandleSet() {
+  for (void *Handle : Handles)
+    FreeLibrary(HMODULE(Handle));
 
-static BOOL CALLBACK
-ELM_Callback(PCSTR ModuleName, DWORD64 ModuleBase,
-             ULONG ModuleSize, PVOID UserContext) {
-  OpenedHandles->insert((HMODULE)ModuleBase);
-  return TRUE;
+  // 'Process' should not be released on Windows.
+  assert((!Process || Process==this) && "Bad Handle");
 }
 
-sys::DynamicLibrary
-sys::DynamicLibrary::getPermanentLibrary(const char *filename,
-                                         std::string *errMsg) {
-  SmartScopedLock<true> lock(*SymbolsMutex);
-
-  if (!filename) {
-    // When no file is specified, enumerate all DLLs and EXEs in the process.
-    if (!fEnumerateLoadedModules) {
-      if (!loadDebugHelp()) {
-        assert(false && "These APIs should always be available");
-        return DynamicLibrary();
-      }
-    }
+void *DynamicLibrary::HandleSet::DLOpen(const char *File, std::string *Err) {
+  // Create the instance and return it to be the *Process* handle
+  // simillar to dlopen(NULL, RTLD_LAZY|RTLD_GLOBAL)
+  if (!File)
+    return &(*OpenedHandles);
 
-    fEnumerateLoadedModules(GetCurrentProcess(), ELM_Callback, 0);
-    // Dummy library that represents "search all handles".
-    // This is mostly to ensure that the return value still shows up as "valid".
-    return DynamicLibrary(&OpenedHandles);
-  }
-
-  SmallVector<wchar_t, MAX_PATH> filenameUnicode;
-  if (std::error_code ec = windows::UTF8ToUTF16(filename, filenameUnicode)) {
+  SmallVector<wchar_t, MAX_PATH> FileUnicode;
+  if (std::error_code ec = windows::UTF8ToUTF16(File, FileUnicode)) {
     SetLastError(ec.value());
-    MakeErrMsg(errMsg, std::string(filename) + ": Can't convert to UTF-16");
-    return DynamicLibrary();
+    MakeErrMsg(Err, std::string(File) + ": Can't convert to UTF-16");
+    return &DynamicLibrary::Invalid;
   }
 
-  HMODULE a_handle = LoadLibraryW(filenameUnicode.data());
-
-  if (a_handle == 0) {
-    MakeErrMsg(errMsg, std::string(filename) + ": Can't open");
-    return DynamicLibrary();
+  HMODULE Handle = LoadLibraryW(FileUnicode.data());
+  if (Handle == NULL) {
+    MakeErrMsg(Err, std::string(File) + ": Can't open");
+    return &DynamicLibrary::Invalid;
   }
 
-  // If we've already loaded this library, FreeLibrary() the handle in order to
-  // keep the internal refcount at +1.
-  if (!OpenedHandles->insert(a_handle).second)
-    FreeLibrary(a_handle);
+  return reinterpret_cast<void*>(Handle);
+}
+
+static DynamicLibrary::HandleSet *IsOpenedHandlesInstance(void *Handle) {
+  if (!OpenedHandles.isConstructed())
+    return false;
+  DynamicLibrary::HandleSet &Inst = *OpenedHandles;
+  return Handle == &Inst ? &Inst : nullptr;
+}
 
-  return DynamicLibrary(a_handle);
+void DynamicLibrary::HandleSet::DLClose(void *Handle) {
+  if (HandleSet* HS = IsOpenedHandlesInstance(Handle))
+    HS->Process = nullptr; // Just drop the *Process* handle.
+  else
+    FreeLibrary((HMODULE)Handle);
 }
 
-sys::DynamicLibrary
-sys::DynamicLibrary::addPermanentLibrary(void *handle, std::string *errMsg) {
-  SmartScopedLock<true> lock(*SymbolsMutex);
-  // If we've already loaded this library, tell the caller.
-  if (!OpenedHandles->insert((HMODULE)handle).second) {
-    MakeErrMsg(errMsg, "Library already loaded");
-    return DynamicLibrary();
+static bool GetProcessModules(HANDLE H, DWORD &Bytes, HMODULE *Data = nullptr) {
+#ifdef _WIN64
+  const DWORD Flags = LIST_MODULES_64BIT;
+#else
+  const DWORD Flags = LIST_MODULES_32BIT;
+#endif
+
+  if (!EnumProcessModulesEx(H, Data, Bytes, &Bytes, Flags)) {
+    std::string Err;
+    if (MakeErrMsg(&Err, "EnumProcessModulesEx failure"))
+      llvm::errs() << Err << "\n";
+    return false;
   }
+  return true;
+}
 
-  return DynamicLibrary(handle);
+void *DynamicLibrary::HandleSet::DLSym(void *Handle, const char *Symbol) {
+  HandleSet* HS = IsOpenedHandlesInstance(Handle);
+  if (!HS)
+    return (void *)uintptr_t(GetProcAddress((HMODULE)Handle, Symbol));
+
+  // Could have done a dlclose on the *Process* handle
+  if (!HS->Process)
+    return nullptr;
+
+  // Trials indicate EnumProcessModulesEx is consistantly faster than using
+  // EnumerateLoadedModules64 or CreateToolhelp32Snapshot.
+  //
+  // | Handles | DbgHelp.dll | CreateSnapshot | EnumProcessModulesEx
+  // |=========|=============|========================================
+  // | 37      | 0.0000585 * | 0.0003031      | 0.0000152
+  // | 1020    | 0.0026310 * | 0.0121598      | 0.0002683
+  // | 2084    | 0.0149418 * | 0.0369936      | 0.0005610
+  //
+  // * Not including the load time of Dbghelp.dll (~.005 sec)
+  //
+  // There's still a case to somehow cache the result of EnumProcessModulesEx
+  // across invocations, but the complication of doing that properly...
+  // Possibly using LdrRegisterDllNotification to invalidate the cache?
+
+  DWORD Bytes = 0;
+  HMODULE Self = HMODULE(GetCurrentProcess());
+  if (!GetProcessModules(Self, Bytes))
+    return nullptr;
+
+  // Get the most recent list in case any modules added/removed between calls
+  // to EnumProcessModulesEx that gets the amount of, then copies the HMODULES.
+  // MSDN is pretty clear that if the module list changes during the call to
+  // EnumProcessModulesEx the results should not be used.
+  std::vector<HMODULE> Handles;
+  do {
+    assert(Bytes && ((Bytes % sizeof(HMODULE)) == 0) &&
+           "Should have at least one module and be aligned");
+    Handles.resize(Bytes / sizeof(HMODULE));
+    if (!GetProcessModules(Self, Bytes, Handles.data()))
+      return nullptr;
+  } while (Bytes != (Handles.size() * sizeof(HMODULE)));
+
+  // Try EXE first, mirroring what dlsym(dlopen(NULL)) does.
+  if (FARPROC Ptr = GetProcAddress(HMODULE(Handles.front()), Symbol))
+    return (void *) uintptr_t(Ptr);
+
+  if (Handles.size() > 1) {
+    // This is different behaviour than what Posix dlsym(dlopen(NULL)) does.
+    // Doing that here is causing real problems for the JIT where msvc.dll
+    // and ucrt.dll can define the same symbols. The runtime linker will choose
+    // symbols from ucrt.dll first, but iterating NOT in reverse here would
+    // mean that the msvc.dll versions would be returned.
+
+    for (auto I = Handles.rbegin(), E = Handles.rend()-1; I != E; ++I) {
+      if (FARPROC Ptr = GetProcAddress(HMODULE(*I), Symbol))
+        return (void *) uintptr_t(Ptr);
+    }
+  }
+  return nullptr;
 }
 
+
 // Stack probing routines are in the support library (e.g. libgcc), but we don't
 // have dynamic linking on windows. Provide a hook.
 #define EXPLICIT_SYMBOL(SYM)                    \
@@ -129,38 +176,18 @@ sys::DynamicLibrary::addPermanentLibrary(void *handle, std::string *errMsg) {
 #undef INLINE_DEF_SYMBOL1
 #undef INLINE_DEF_SYMBOL2
 
-void *sys::DynamicLibrary::SearchForAddressOfSymbol(const char *symbolName) {
-  SmartScopedLock<true> Lock(*SymbolsMutex);
-
-  // First check symbols added via AddSymbol().
-  if (ExplicitSymbols.isConstructed()) {
-    StringMap<void *>::iterator i = ExplicitSymbols->find(symbolName);
-
-    if (i != ExplicitSymbols->end())
-      return i->second;
-  }
-
-  // Now search the libraries.
-  if (OpenedHandles.isConstructed()) {
-    for (DenseSet<HMODULE>::iterator I = OpenedHandles->begin(),
-         E = OpenedHandles->end(); I != E; ++I) {
-      FARPROC ptr = GetProcAddress((HMODULE)*I, symbolName);
-      if (ptr) {
-        return (void *)(intptr_t)ptr;
-      }
-    }
-  }
+static void *DoSearch(const char *SymbolName) {
 
 #define EXPLICIT_SYMBOL(SYM)                                                   \
-  if (!strcmp(symbolName, #SYM))                                               \
+  if (!strcmp(SymbolName, #SYM))                                               \
     return (void *)&SYM;
 #define EXPLICIT_SYMBOL2(SYMFROM, SYMTO)                                       \
-  if (!strcmp(symbolName, #SYMFROM))                                           \
+  if (!strcmp(SymbolName, #SYMFROM))                                           \
     return (void *)&SYMTO;
 
 #ifdef _M_IX86
 #define INLINE_DEF_SYMBOL1(TYP, SYM)                                           \
-  if (!strcmp(symbolName, #SYM))                                               \
+  if (!strcmp(SymbolName, #SYM))                                               \
     return (void *)&inline_##SYM;
 #define INLINE_DEF_SYMBOL2(TYP, SYM) INLINE_DEF_SYMBOL1(TYP, SYM)
 #endif
@@ -174,15 +201,5 @@ void *sys::DynamicLibrary::SearchForAddressOfSymbol(const char *symbolName) {
 #undef INLINE_DEF_SYMBOL1
 #undef INLINE_DEF_SYMBOL2
 
-  return 0;
-}
-
-void *sys::DynamicLibrary::getAddressOfSymbol(const char *symbolName) {
-  if (!isValid())
-    return NULL;
-  if (Data == &OpenedHandles)
-    return SearchForAddressOfSymbol(symbolName);
-  return (void *)(intptr_t)GetProcAddress((HMODULE)Data, symbolName);
-}
-
+  return nullptr;
 }

unittests/Support/CMakeLists.txt

@@ -67,3 +67,5 @@ add_llvm_unittest(SupportTests
 
 # ManagedStatic.cpp uses <pthread>.
 target_link_libraries(SupportTests ${LLVM_PTHREAD_LIB})
+
+add_subdirectory(DynamicLibrary)

unittests/Support/DynamicLibrary/CMakeLists.txt

@@ -0,0 +1,19 @@
+set(LLVM_LINK_COMPONENTS Support)
+
+add_llvm_unittest(DynamicLibraryTests DynamicLibraryTest.cpp)
+
+export_executable_symbols(DynamicLibraryTests)
+
+add_library(PipSqueak SHARED PipSqueak.cxx)
+
+set_output_directory(PipSqueak
+  BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_CFG_INTDIR}
+  LIBRARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_CFG_INTDIR}
+  )
+
+set_target_properties(PipSqueak
+  PROPERTIES PREFIX ""
+  SUFFIX ".so"
+  )
+
+add_dependencies(DynamicLibraryTests PipSqueak)

unittests/Support/DynamicLibrary/DynamicLibraryTest.cpp

@@ -0,0 +1,133 @@
+//===- llvm/unittest/Support/DynamicLibrary/DynamicLibraryTest.cpp --------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Config/config.h"
+#include "llvm/Support/DynamicLibrary.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/ManagedStatic.h"
+#include "llvm/Support/Path.h"
+#include "gtest/gtest.h"
+
+#include "PipSqueak.h"
+#include <string>
+
+using namespace llvm;
+using namespace llvm::sys;
+
+extern "C" PIPSQUEAK_EXPORT const char *TestA() { return "ProcessCall"; }
+
+std::string LibPath() {
+  std::string Path =
+      fs::getMainExecutable("DynamicLibraryTests", (void *)&TestA);
+  llvm::SmallString<256> Buf(path::parent_path(Path));
+  path::append(Buf, "PipSqueak.so");
+  return Buf.str();
+}
+
+#if defined(_WIN32) || (defined(HAVE_DLFCN_H) && defined(HAVE_DLOPEN))
+
+typedef void (*SetStrings)(std::string &GStr, std::string &LStr);
+typedef const char *(*GetString)();
+
+template <class T> static T FuncPtr(void *Ptr) {
+  union {
+    T F;
+    void *P;
+  } Tmp;
+  Tmp.P = Ptr;
+  return Tmp.F;
+}
+template <class T> static void* PtrFunc(T *Func) {
+  union {
+    T *F;
+    void *P;
+  } Tmp;
+  Tmp.F = Func;
+  return Tmp.P;
+}
+
+static const char *OverloadTestA() { return "OverloadCall"; }
+
+std::string StdString(const char *Ptr) { return Ptr ? Ptr : ""; }
+
+TEST(DynamicLibrary, Overload) {
+  {
+    std::string Err;
+    llvm_shutdown_obj Shutdown;
+    DynamicLibrary DL =
+        DynamicLibrary::getPermanentLibrary(LibPath().c_str(), &Err);
+    EXPECT_TRUE(DL.isValid());
+    EXPECT_TRUE(Err.empty());
+
+    GetString GS = FuncPtr<GetString>(DL.getAddressOfSymbol("TestA"));
+    EXPECT_TRUE(GS != nullptr && GS != &TestA);
+    EXPECT_EQ(StdString(GS()), "LibCall");
+
+    GS = FuncPtr<GetString>(DynamicLibrary::SearchForAddressOfSymbol("TestA"));
+    EXPECT_TRUE(GS != nullptr && GS != &TestA);
+    EXPECT_EQ(StdString(GS()), "LibCall");
+
+    DL = DynamicLibrary::getPermanentLibrary(nullptr, &Err);
+    EXPECT_TRUE(DL.isValid());
+    EXPECT_TRUE(Err.empty());
+
+    GS = FuncPtr<GetString>(DynamicLibrary::SearchForAddressOfSymbol("TestA"));
+    EXPECT_TRUE(GS != nullptr && GS == &TestA);
+    EXPECT_EQ(StdString(GS()), "ProcessCall");
+
+    GS = FuncPtr<GetString>(DL.getAddressOfSymbol("TestA"));
+    EXPECT_TRUE(GS != nullptr && GS == &TestA);
+    EXPECT_EQ(StdString(GS()), "ProcessCall");
+
+    DynamicLibrary::AddSymbol("TestA", PtrFunc(&OverloadTestA));
+    GS = FuncPtr<GetString>(DL.getAddressOfSymbol("TestA"));
+    EXPECT_TRUE(GS != nullptr && GS != &OverloadTestA);
+
+    GS = FuncPtr<GetString>(DynamicLibrary::SearchForAddressOfSymbol("TestA"));
+    EXPECT_TRUE(GS != nullptr && GS == &OverloadTestA);
+    EXPECT_EQ(StdString(GS()), "OverloadCall");
+  }
+  EXPECT_TRUE(FuncPtr<GetString>(DynamicLibrary::SearchForAddressOfSymbol(
+                  "TestA")) == nullptr);
+}
+
+TEST(DynamicLibrary, Shutdown) {
+  std::string A, B;
+  {
+    std::string Err;
+    llvm_shutdown_obj Shutdown;
+    DynamicLibrary DL =
+        DynamicLibrary::getPermanentLibrary(LibPath().c_str(), &Err);
+    EXPECT_TRUE(DL.isValid());
+    EXPECT_TRUE(Err.empty());
+
+    SetStrings SS = FuncPtr<SetStrings>(
+        DynamicLibrary::SearchForAddressOfSymbol("SetStrings"));
+    EXPECT_TRUE(SS != nullptr);
+
+    SS(A, B);
+    EXPECT_EQ(B, "Local::Local");
+  }
+  EXPECT_EQ(A, "Global::~Global");
+  EXPECT_EQ(B, "Local::~Local");
+  EXPECT_TRUE(FuncPtr<SetStrings>(DynamicLibrary::SearchForAddressOfSymbol(
+                  "SetStrings")) == nullptr);
+}
+
+#else
+
+TEST(DynamicLibrary, Unsupported) {
+  std::string Err;
+  DynamicLibrary DL =
+      DynamicLibrary::getPermanentLibrary(LibPath().c_str(), &Err);
+  EXPECT_FALSE(DL.isValid());
+  EXPECT_EQ(Err, "dlopen() not supported on this platform");
+}
+
+#endif

unittests/Support/DynamicLibrary/PipSqueak.cxx

@@ -0,0 +1,36 @@
+//===- llvm/unittest/Support/DynamicLibrary/PipSqueak.cxx -----------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "PipSqueak.h"
+#include <string>
+
+struct Global {
+  std::string *Str;
+  Global() : Str(nullptr) {}
+  ~Global() {
+    if (Str)
+      *Str = "Global::~Global";
+  }
+};
+
+struct Local {
+  std::string &Str;
+  Local(std::string &S) : Str(S) { Str = "Local::Local"; }
+  ~Local() { Str = "Local::~Local"; }
+};
+
+static Global Glb;
+
+extern "C" PIPSQUEAK_EXPORT void SetStrings(std::string &GStr,
+                                            std::string &LStr) {
+  static Local Lcl(LStr);
+  Glb.Str = &GStr;
+}
+
+extern "C" PIPSQUEAK_EXPORT const char *TestA() { return "LibCall"; }

unittests/Support/DynamicLibrary/PipSqueak.h

@@ -0,0 +1,19 @@
+//===- llvm/unittest/Support/DynamicLibrary/PipSqueak.h -------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_PIPSQUEAK_H
+#define LLVM_PIPSQUEAK_H
+
+#ifdef _WIN32
+#define PIPSQUEAK_EXPORT __declspec(dllexport)
+#else
+#define PIPSQUEAK_EXPORT
+#endif
+
+#endif