| 1 |
/*========================== begin_copyright_notice ============================ |
| 2 |
|
| 3 |
Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 |
See https://llvm.org/LICENSE.txt for license information. |
| 5 |
SPDX-License-Identifier: Apache-2.0 with LLVM-exception |
| 6 |
|
| 7 |
============================= end_copyright_notice ===========================*/ |
| 8 |
|
| 9 |
From 564481aebe18a723c9cfe9ea9ca5808771f7e9d8 Mon Sep 17 00:00:00 2001 |
| 10 |
From: Andrew Ng <[email protected]> |
| 11 |
Date: Sat, 16 Mar 2019 19:36:29 +0000 |
| 12 |
Subject: [PATCH] [Support] ThreadPoolExecutor fixes for Windows/MinGW |
| 13 |
|
| 14 |
Changed ThreadPoolExecutor to no longer use detached threads and instead |
| 15 |
to join threads on destruction. This is to prevent intermittent crashing |
| 16 |
on Windows when doing a normal full exit, e.g. via exit(). |
| 17 |
|
| 18 |
Changed ThreadPoolExecutor to be a ManagedStatic so that it can be |
| 19 |
stopped on llvm_shutdown(). Without this, it would only be stopped in |
| 20 |
the destructor when doing a full exit. This is required to avoid |
| 21 |
intermittent crashing on Windows due to a race condition between the |
| 22 |
ThreadPoolExecutor starting up threads and the process doing a fast |
| 23 |
exit, e.g. via _exit(). |
| 24 |
|
| 25 |
The Windows crashes appear to only occur with the MSVC static runtimes |
| 26 |
and are more frequent with the debug static runtime. |
| 27 |
|
| 28 |
These changes also prevent intermittent deadlocks on exit with the MinGW |
| 29 |
runtime. |
| 30 |
|
| 31 |
Differential Revision: https://reviews.llvm.org/D70447 |
| 32 |
--- |
| 33 |
llvm/lib/Support/Parallel.cpp | 82 +++++++++++++++++++++++++++-------- |
| 34 |
1 file changed, 64 insertions(+), 18 deletions(-) |
| 35 |
|
| 36 |
diff --git a/llvm/lib/Support/Parallel.cpp b/llvm/lib/Support/Parallel.cpp |
| 37 |
index 355c64b7d0793..523665d14b029 100644 |
| 38 |
--- a/llvm/lib/Support/Parallel.cpp |
| 39 |
+++ b/llvm/lib/Support/Parallel.cpp |
| 40 |
@@ -8,14 +8,17 @@ |
| 41 |
|
| 42 |
#include "llvm/Support/Parallel.h" |
| 43 |
#include "llvm/Config/llvm-config.h" |
| 44 |
+#include "llvm/Support/ManagedStatic.h" |
| 45 |
|
| 46 |
#if LLVM_ENABLE_THREADS |
| 47 |
|
| 48 |
#include "llvm/Support/Threading.h" |
| 49 |
|
| 50 |
#include <atomic> |
| 51 |
+#include <future> |
| 52 |
#include <stack> |
| 53 |
#include <thread> |
| 54 |
+#include <vector> |
| 55 |
|
| 56 |
namespace llvm { |
| 57 |
namespace parallel { |
| 58 |
@@ -36,30 +39,53 @@ class Executor { |
| 59 |
/// in filo order. |
| 60 |
class ThreadPoolExecutor : public Executor { |
| 61 |
public: |
| 62 |
- explicit ThreadPoolExecutor(unsigned ThreadCount = hardware_concurrency()) |
| 63 |
- : Done(ThreadCount) { |
| 64 |
+ explicit ThreadPoolExecutor(unsigned ThreadCount = hardware_concurrency()) { |
| 65 |
// Spawn all but one of the threads in another thread as spawning threads |
| 66 |
// can take a while. |
| 67 |
- std::thread([&, ThreadCount] { |
| 68 |
- for (size_t i = 1; i < ThreadCount; ++i) { |
| 69 |
- std::thread([=] { work(); }).detach(); |
| 70 |
+ Threads.reserve(ThreadCount); |
| 71 |
+ Threads.resize(1); |
| 72 |
+ std::lock_guard<std::mutex> Lock(Mutex); |
| 73 |
+ Threads[0] = std::thread([&, ThreadCount] { |
| 74 |
+ for (unsigned i = 1; i < ThreadCount; ++i) { |
| 75 |
+ Threads.emplace_back([=] { work(); }); |
| 76 |
+ if (Stop) |
| 77 |
+ break; |
| 78 |
} |
| 79 |
+ ThreadsCreated.set_value(); |
| 80 |
work(); |
| 81 |
- }).detach(); |
| 82 |
+ }); |
| 83 |
} |
| 84 |
|
| 85 |
- ~ThreadPoolExecutor() override { |
| 86 |
- std::unique_lock<std::mutex> Lock(Mutex); |
| 87 |
- Stop = true; |
| 88 |
- Lock.unlock(); |
| 89 |
+ void stop() { |
| 90 |
+ { |
| 91 |
+ std::lock_guard<std::mutex> Lock(Mutex); |
| 92 |
+ if (Stop) |
| 93 |
+ return; |
| 94 |
+ Stop = true; |
| 95 |
+ } |
| 96 |
Cond.notify_all(); |
| 97 |
- // Wait for ~Latch. |
| 98 |
+ ThreadsCreated.get_future().wait(); |
| 99 |
} |
| 100 |
|
| 101 |
+ ~ThreadPoolExecutor() override { |
| 102 |
+ stop(); |
| 103 |
+ std::thread::id CurrentThreadId = std::this_thread::get_id(); |
| 104 |
+ for (std::thread &T : Threads) |
| 105 |
+ if (T.get_id() == CurrentThreadId) |
| 106 |
+ T.detach(); |
| 107 |
+ else |
| 108 |
+ T.join(); |
| 109 |
+ } |
| 110 |
+ |
| 111 |
+ struct Deleter { |
| 112 |
+ static void call(void *Ptr) { ((ThreadPoolExecutor *)Ptr)->stop(); } |
| 113 |
+ }; |
| 114 |
+ |
| 115 |
void add(std::function<void()> F) override { |
| 116 |
- std::unique_lock<std::mutex> Lock(Mutex); |
| 117 |
- WorkStack.push(F); |
| 118 |
- Lock.unlock(); |
| 119 |
+ { |
| 120 |
+ std::lock_guard<std::mutex> Lock(Mutex); |
| 121 |
+ WorkStack.push(F); |
| 122 |
+ } |
| 123 |
Cond.notify_one(); |
| 124 |
} |
| 125 |
|
| 126 |
@@ -75,19 +101,39 @@ class ThreadPoolExecutor : public Executor { |
| 127 |
Lock.unlock(); |
| 128 |
Task(); |
| 129 |
} |
| 130 |
- Done.dec(); |
| 131 |
} |
| 132 |
|
| 133 |
std::atomic<bool> Stop{false}; |
| 134 |
std::stack<std::function<void()>> WorkStack; |
| 135 |
std::mutex Mutex; |
| 136 |
std::condition_variable Cond; |
| 137 |
- parallel::detail::Latch Done; |
| 138 |
+ std::promise<void> ThreadsCreated; |
| 139 |
+ std::vector<std::thread> Threads; |
| 140 |
}; |
| 141 |
|
| 142 |
Executor *Executor::getDefaultExecutor() { |
| 143 |
- static ThreadPoolExecutor exec; |
| 144 |
- return &exec; |
| 145 |
+ // The ManagedStatic enables the ThreadPoolExecutor to be stopped via |
| 146 |
+ // llvm_shutdown() which allows a "clean" fast exit, e.g. via _exit(). This |
| 147 |
+ // stops the thread pool and waits for any worker thread creation to complete |
| 148 |
+ // but does not wait for the threads to finish. The wait for worker thread |
| 149 |
+ // creation to complete is important as it prevents intermittent crashes on |
| 150 |
+ // Windows due to a race condition between thread creation and process exit. |
| 151 |
+ // |
| 152 |
+ // The ThreadPoolExecutor will only be destroyed when the static unique_ptr to |
| 153 |
+ // it is destroyed, i.e. in a normal full exit. The ThreadPoolExecutor |
| 154 |
+ // destructor ensures it has been stopped and waits for worker threads to |
| 155 |
+ // finish. The wait is important as it prevents intermittent crashes on |
| 156 |
+ // Windows when the process is doing a full exit. |
| 157 |
+ // |
| 158 |
+ // The Windows crashes appear to only occur with the MSVC static runtimes and |
| 159 |
+ // are more frequent with the debug static runtime. |
| 160 |
+ // |
| 161 |
+ // This also prevents intermittent deadlocks on exit with the MinGW runtime. |
| 162 |
+ static ManagedStatic<ThreadPoolExecutor, object_creator<ThreadPoolExecutor>, |
| 163 |
+ ThreadPoolExecutor::Deleter> |
| 164 |
+ ManagedExec; |
| 165 |
+ static std::unique_ptr<ThreadPoolExecutor> Exec(&(*ManagedExec)); |
| 166 |
+ return Exec.get(); |
| 167 |
} |
| 168 |
} // namespace |
| 169 |
|