Debian Codegraph @

extra/gperftools/gperftools-2.15/src/windows/patch_functions.cc

raw
1 // -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*-
2 // Copyright (c) 2007, Google Inc.
3 // All rights reserved.
4 //
5 // Redistribution and use in source and binary forms, with or without
6 // modification, are permitted provided that the following conditions are
7 // met:
8 //
9 // * Redistributions of source code must retain the above copyright
10 // notice, this list of conditions and the following disclaimer.
11 // * Redistributions in binary form must reproduce the above
12 // copyright notice, this list of conditions and the following disclaimer
13 // in the documentation and/or other materials provided with the
14 // distribution.
15 // * Neither the name of Google Inc. nor the names of its
16 // contributors may be used to endorse or promote products derived from
17 // this software without specific prior written permission.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 //
31 // ---
32 // Author: Craig Silverstein
33 //
34 // The main purpose of this file is to patch the libc allocation
35 // routines (malloc and friends, but also _msize and other
36 // windows-specific libc-style routines). However, we also patch
37 // windows routines to do accounting. We do better at the former than
38 // the latter. Here are some comments from Paul Pluzhnikov about what
39 // it might take to do a really good job patching windows routines to
40 // keep track of memory usage:
41 //
42 // "You should intercept at least the following:
43 // HeapCreate HeapDestroy HeapAlloc HeapReAlloc HeapFree
44 // RtlCreateHeap RtlDestroyHeap RtlAllocateHeap RtlFreeHeap
45 // malloc calloc realloc free
46 // malloc_dbg calloc_dbg realloc_dbg free_dbg
47 // Some of these call the other ones (but not always), sometimes
48 // recursively (i.e. HeapCreate may call HeapAlloc on a different
49 // heap, IIRC)."
50 //
51 // Since Paul didn't mention VirtualAllocEx, he may not have even been
52 // considering all the mmap-like functions that windows has (or he may
53 // just be ignoring it because he's seen we already patch it). Of the
54 // above, we do not patch the *_dbg functions, and of the windows
55 // functions, we only patch HeapAlloc and HeapFree.
56 //
57 // The *_dbg functions come into play with /MDd, /MTd, and /MLd,
58 // probably. It may be ok to just turn off tcmalloc in those cases --
59 // if the user wants the windows debug malloc, they probably don't
60 // want tcmalloc! We should also test with all of /MD, /MT, and /ML,
61 // which we're not currently doing.
62
63 // TODO(csilvers): try to do better here? Paul does conclude:
64 // "Keeping track of all of this was a nightmare."
65
66 #ifndef _WIN32
67 # error You should only be including windows/patch_functions.cc in a windows environment!
68 #endif
69
70 #include <config.h>
71
72 #ifdef WIN32_OVERRIDE_ALLOCATORS
73 #error This file is intended for patching allocators - use override_functions.cc instead.
74 #endif
75
76 // We use psapi. Non-MSVC systems will have to link this in themselves.
77 #ifdef _MSC_VER
78 #pragma comment(lib, "Psapi.lib")
79 #endif
80
81 // Make sure we always use the 'old' names of the psapi functions.
82 #ifndef PSAPI_VERSION
83 #define PSAPI_VERSION 1
84 #endif
85
86 #include <windows.h>
87 #include <stdio.h>
88 #include <malloc.h> // for _msize and _expand
89 #include <psapi.h> // for EnumProcessModules, GetModuleInformation, etc.
90 #include <set>
91 #include <map>
92 #include <vector>
93 #include <base/logging.h>
94 #include "base/spinlock.h"
95 #include "getenv_safe.h" // for TCMallocGetenvSafe
96 #include "gperftools/malloc_hook.h"
97 #include "malloc_hook-inl.h"
98 #include "preamble_patcher.h"
99
100 // The maximum number of modules we allow to be in one executable
101 const int kMaxModules = 8182;
102
103 // These are hard-coded, unfortunately. :-( They are also probably
104 // compiler specific. See get_mangled_names.cc, in this directory,
105 // for instructions on how to update these names for your compiler.
106 #ifdef _WIN64
107 const char kMangledNew[] = "??2@YAPEAX_K@Z";
108 const char kMangledNewArray[] = "??_U@YAPEAX_K@Z";
109 const char kMangledDelete[] = "??3@YAXPEAX@Z";
110 const char kMangledDeleteArray[] = "??_V@YAXPEAX@Z";
111 const char kMangledNewNothrow[] = "??2@YAPEAX_KAEBUnothrow_t@std@@@Z";
112 const char kMangledNewArrayNothrow[] = "??_U@YAPEAX_KAEBUnothrow_t@std@@@Z";
113 const char kMangledDeleteNothrow[] = "??3@YAXPEAXAEBUnothrow_t@std@@@Z";
114 const char kMangledDeleteArrayNothrow[] = "??_V@YAXPEAXAEBUnothrow_t@std@@@Z";
115 #else
116 const char kMangledNew[] = "??2@YAPAXI@Z";
117 const char kMangledNewArray[] = "??_U@YAPAXI@Z";
118 const char kMangledDelete[] = "??3@YAXPAX@Z";
119 const char kMangledDeleteArray[] = "??_V@YAXPAX@Z";
120 const char kMangledNewNothrow[] = "??2@YAPAXIABUnothrow_t@std@@@Z";
121 const char kMangledNewArrayNothrow[] = "??_U@YAPAXIABUnothrow_t@std@@@Z";
122 const char kMangledDeleteNothrow[] = "??3@YAXPAXABUnothrow_t@std@@@Z";
123 const char kMangledDeleteArrayNothrow[] = "??_V@YAXPAXABUnothrow_t@std@@@Z";
124 #endif
125
126 // This is an unused but exported symbol that we can use to tell the
127 // MSVC linker to bring in libtcmalloc, via the /INCLUDE linker flag.
128 // Without this, the linker will likely decide that libtcmalloc.dll
129 // doesn't add anything to the executable (since it does all its work
130 // through patching, which the linker can't see), and ignore it
131 // entirely. (The name 'tcmalloc' is already reserved for a
132 // namespace. I'd rather export a variable named "_tcmalloc", but I
133 // couldn't figure out how to get that to work. This function exports
134 // the symbol "__tcmalloc".)
135 extern "C" PERFTOOLS_DLL_DECL void _tcmalloc();
136 void _tcmalloc() { }
137
138 // This is the version needed for windows x64, which has a different
139 // decoration scheme which doesn't auto-add a leading underscore.
140 extern "C" PERFTOOLS_DLL_DECL void __tcmalloc();
141 void __tcmalloc() { }
142
143 namespace { // most everything here is in an unnamed namespace
144
145 typedef void (*GenericFnPtr)();
146
147 using sidestep::PreamblePatcher;
148
149 struct ModuleEntryCopy; // defined below
150
151 // These functions are how we override the memory allocation
152 // functions, just like tcmalloc.cc and malloc_hook.cc do.
153
154 // This is information about the routines we're patching, for a given
155 // module that implements libc memory routines. A single executable
156 // can have several libc implementations running about (in different
157 // .dll's), and we need to patch/unpatch them all. This defines
158 // everything except the new functions we're patching in, which
159 // are defined in LibcFunctions, below.
160 class LibcInfo {
161 public:
162 LibcInfo() {
163 memset(this, 0, sizeof(*this)); // easiest way to initialize the array
164 }
165
166 bool patched() const { return is_valid(); }
167 void set_is_valid(bool b) { is_valid_ = b; }
168 // According to http://msdn.microsoft.com/en-us/library/ms684229(VS.85).aspx:
169 // "The load address of a module (lpBaseOfDll) is the same as the HMODULE
170 // value."
171 HMODULE hmodule() const {
172 return reinterpret_cast<HMODULE>(const_cast<void*>(module_base_address_));
173 }
174
175 // Populates all the windows_fn_[] vars based on our module info.
176 // Returns false if windows_fn_ is all NULL's, because there's
177 // nothing to patch. Also populates the rest of the module_entry
178 // info, such as the module's name.
179 bool PopulateWindowsFn(const ModuleEntryCopy& module_entry);
180
181 protected:
182 void CopyFrom(const LibcInfo& that) {
183 if (this == &that)
184 return;
185 this->is_valid_ = that.is_valid_;
186 memcpy(this->windows_fn_, that.windows_fn_, sizeof(windows_fn_));
187 this->module_base_address_ = that.module_base_address_;
188 this->module_base_size_ = that.module_base_size_;
189 }
190
191 enum {
192 kMalloc, kFree, kRealloc, kCalloc,
193 kNew, kNewArray, kDelete, kDeleteArray,
194 kNewNothrow, kNewArrayNothrow, kDeleteNothrow, kDeleteArrayNothrow,
195 // These are windows-only functions from malloc.h
196 k_Msize, k_Expand,
197 // A MS CRT "internal" function, implemented using _calloc_impl
198 k_CallocCrt,
199 // Underlying deallocation functions called by CRT internal functions or operator delete
200 kFreeBase, kFreeDbg,
201 kNumFunctions
202 };
203
204 // I'd like to put these together in a struct (perhaps in the
205 // subclass, so we can put in perftools_fn_ as well), but vc8 seems
206 // to have a bug where it doesn't initialize the struct properly if
207 // we try to take the address of a function that's not yet loaded
208 // from a dll, as is the common case for static_fn_. So we need
209 // each to be in its own array. :-(
210 static const char* const function_name_[kNumFunctions];
211
212 // This function is only used when statically linking the binary.
213 // In that case, loading malloc/etc from the dll (via
214 // PatchOneModule) won't work, since there are no dlls. Instead,
215 // you just want to be taking the address of malloc/etc directly.
216 // In the common, non-static-link case, these pointers will all be
217 // NULL, since this initializer runs before msvcrt.dll is loaded.
218 static const GenericFnPtr static_fn_[kNumFunctions];
219
220 // This is the address of the function we are going to patch
221 // (malloc, etc). Other info about the function is in the
222 // patch-specific subclasses, below.
223 GenericFnPtr windows_fn_[kNumFunctions];
224
225 // This is set to true when this structure is initialized (because
226 // we're patching a new library) and set to false when it's
227 // uninitialized (because we've freed that library).
228 bool is_valid_;
229
230 const void *module_base_address_;
231 size_t module_base_size_;
232
233 public:
234 // These shouldn't have to be public, since only subclasses of
235 // LibcInfo need it, but they do. Maybe something to do with
236 // templates. Shrug. I hide them down here so users won't see
237 // them. :-) (OK, I also need to define ctrgProcAddress late.)
238 bool is_valid() const { return is_valid_; }
239 GenericFnPtr windows_fn(int ifunction) const {
240 return windows_fn_[ifunction];
241 }
242 // These three are needed by ModuleEntryCopy.
243 static const int ctrgProcAddress = kNumFunctions;
244 static GenericFnPtr static_fn(int ifunction) {
245 return static_fn_[ifunction];
246 }
247 static const char* const function_name(int ifunction) {
248 return function_name_[ifunction];
249 }
250 };
251
252 // Template trickiness: logically, a LibcInfo would include
253 // Windows_malloc_, origstub_malloc_, and Perftools_malloc_: for a
254 // given module, these three go together. And in fact,
255 // Perftools_malloc_ may need to call origstub_malloc_, which means we
256 // either need to change Perftools_malloc_ to take origstub_malloc_ as
257 // an argument -- unfortunately impossible since it needs to keep the
258 // same API as normal malloc -- or we need to write a different
259 // version of Perftools_malloc_ for each LibcInfo instance we create.
260 // We choose the second route, and use templates to implement it (we
261 // could have also used macros). So to get multiple versions
262 // of the struct, we say "struct<1> var1; struct<2> var2;". The price
263 // we pay is some code duplication, and more annoying, each instance
264 // of this var is a separate type.
265 template<int> class LibcInfoWithPatchFunctions : public LibcInfo {
266 public:
267 // me_info should have had PopulateWindowsFn() called on it, so the
268 // module_* vars and windows_fn_ are set up.
269 bool Patch(const LibcInfo& me_info);
270 void Unpatch();
271
272 private:
273 // This holds the original function contents after we patch the function.
274 // This has to be defined static in the subclass, because the perftools_fns
275 // reference origstub_fn_.
276 static GenericFnPtr origstub_fn_[kNumFunctions];
277
278 // This is the function we want to patch in
279 static const GenericFnPtr perftools_fn_[kNumFunctions];
280
281 static void* Perftools_malloc(size_t size) __THROW;
282 static void Perftools_free(void* ptr) __THROW;
283 static void Perftools_free_base(void* ptr) __THROW;
284 static void Perftools_free_dbg(void* ptr, int block_use) __THROW;
285 static void* Perftools_realloc(void* ptr, size_t size) __THROW;
286 static void* Perftools_calloc(size_t nmemb, size_t size) __THROW;
287 static void* Perftools_new(size_t size);
288 static void* Perftools_newarray(size_t size);
289 static void Perftools_delete(void *ptr);
290 static void Perftools_deletearray(void *ptr);
291 static void* Perftools_new_nothrow(size_t size,
292 const std::nothrow_t&) __THROW;
293 static void* Perftools_newarray_nothrow(size_t size,
294 const std::nothrow_t&) __THROW;
295 static void Perftools_delete_nothrow(void *ptr,
296 const std::nothrow_t&) __THROW;
297 static void Perftools_deletearray_nothrow(void *ptr,
298 const std::nothrow_t&) __THROW;
299 static size_t Perftools__msize(void *ptr) __THROW;
300 static void* Perftools__expand(void *ptr, size_t size) __THROW;
301 // malloc.h also defines these functions:
302 // _aligned_malloc, _aligned_free,
303 // _recalloc, _aligned_offset_malloc, _aligned_realloc, _aligned_recalloc
304 // _aligned_offset_realloc, _aligned_offset_recalloc, _malloca, _freea
305 // But they seem pretty obscure, and I'm fine not overriding them for now.
306 // It may be they all call into malloc/free anyway.
307 };
308
309 // This is a subset of MODDULEENTRY32, that we need for patching.
310 struct ModuleEntryCopy {
311 LPVOID modBaseAddr; // the same as hmodule
312 DWORD modBaseSize;
313 // This is not part of MODDULEENTRY32, but is needed to avoid making
314 // windows syscalls while we're holding patch_all_modules_lock (see
315 // lock-inversion comments at patch_all_modules_lock definition, below).
316 GenericFnPtr rgProcAddresses[LibcInfo::ctrgProcAddress];
317
318 ModuleEntryCopy() {
319 modBaseAddr = NULL;
320 modBaseSize = 0;
321 for (int i = 0; i < sizeof(rgProcAddresses)/sizeof(*rgProcAddresses); i++)
322 rgProcAddresses[i] = LibcInfo::static_fn(i);
323 }
324 ModuleEntryCopy(const MODULEINFO& mi) {
325 this->modBaseAddr = mi.lpBaseOfDll;
326 this->modBaseSize = mi.SizeOfImage;
327 LPVOID modEndAddr = (char*)mi.lpBaseOfDll + mi.SizeOfImage;
328 for (int i = 0; i < sizeof(rgProcAddresses)/sizeof(*rgProcAddresses); i++) {
329 FARPROC target = ::GetProcAddress(
330 reinterpret_cast<const HMODULE>(mi.lpBaseOfDll),
331 LibcInfo::function_name(i));
332 void* target_addr = reinterpret_cast<void*>(target);
333 // Sometimes a DLL forwards a function to a function in another
334 // DLL. We don't want to patch those forwarded functions --
335 // they'll get patched when the other DLL is processed.
336 if (modBaseAddr <= target_addr && target_addr < modEndAddr)
337 rgProcAddresses[i] = (GenericFnPtr)target;
338 else
339 rgProcAddresses[i] = (GenericFnPtr)NULL;
340 }
341 }
342 };
343
344 // This class is easier because there's only one of them.
345 class WindowsInfo {
346 public:
347 void Patch();
348 void Unpatch();
349
350 private:
351 // TODO(csilvers): should we be patching GlobalAlloc/LocalAlloc instead,
352 // for pre-XP systems?
353 enum {
354 kHeapAlloc, kHeapFree, kLoadLibraryExW, kFreeLibrary,
355 kNumFunctions
356 };
357
358 struct FunctionInfo {
359 const char* const name; // name of fn in a module (eg "malloc")
360 GenericFnPtr windows_fn; // the fn whose name we call (&malloc)
361 GenericFnPtr origstub_fn; // original fn contents after we patch
362 const GenericFnPtr perftools_fn; // fn we want to patch in
363 };
364
365 static FunctionInfo function_info_[kNumFunctions];
366
367 // A Windows-API equivalent of malloc and free
368 static LPVOID WINAPI Perftools_HeapAlloc(HANDLE hHeap, DWORD dwFlags,
369 DWORD_PTR dwBytes);
370 static BOOL WINAPI Perftools_HeapFree(HANDLE hHeap, DWORD dwFlags,
371 LPVOID lpMem);
372 // We don't need the other 3 variants because they all call this one. */
373 static HMODULE WINAPI Perftools_LoadLibraryExW(LPCWSTR lpFileName,
374 HANDLE hFile,
375 DWORD dwFlags);
376 static BOOL WINAPI Perftools_FreeLibrary(HMODULE hLibModule);
377 };
378
379 // If you run out, just add a few more to the array. You'll also need
380 // to update the switch statement in PatchOneModule(), and the list in
381 // UnpatchWindowsFunctions().
382 // main_executable and main_executable_windows are two windows into
383 // the same executable. One is responsible for patching the libc
384 // routines that live in the main executable (if any) to use tcmalloc;
385 // the other is responsible for patching the windows routines like
386 // HeapAlloc/etc to use tcmalloc.
387 static LibcInfoWithPatchFunctions<0> main_executable;
388 static LibcInfoWithPatchFunctions<1> libc1;
389 static LibcInfoWithPatchFunctions<2> libc2;
390 static LibcInfoWithPatchFunctions<3> libc3;
391 static LibcInfoWithPatchFunctions<4> libc4;
392 static LibcInfoWithPatchFunctions<5> libc5;
393 static LibcInfoWithPatchFunctions<6> libc6;
394 static LibcInfoWithPatchFunctions<7> libc7;
395 static LibcInfoWithPatchFunctions<8> libc8;
396 static LibcInfo* g_module_libcs[] = {
397 &libc1, &libc2, &libc3, &libc4, &libc5, &libc6, &libc7, &libc8
398 };
399 static WindowsInfo main_executable_windows;
400
401 const char* const LibcInfo::function_name_[] = {
402 "malloc", "free", "realloc", "calloc",
403 kMangledNew, kMangledNewArray, kMangledDelete, kMangledDeleteArray,
404 // Ideally we should patch the nothrow versions of new/delete, but
405 // at least in msvcrt, nothrow-new machine-code is of a type we
406 // can't patch. Since these are relatively rare, I'm hoping it's ok
407 // not to patch them. (NULL name turns off patching.)
408 NULL, // kMangledNewNothrow,
409 NULL, // kMangledNewArrayNothrow,
410 NULL, // kMangledDeleteNothrow,
411 NULL, // kMangledDeleteArrayNothrow,
412 "_msize", "_expand", "_calloc_crt", "_free_base", "_free_dbg"
413 };
414
415 // For mingw, I can't patch the new/delete here, because the
416 // instructions are too small to patch. Luckily, they're so small
417 // because all they do is call into malloc/free, so they still end up
418 // calling tcmalloc routines, and we don't actually lose anything
419 // (except maybe some stacktrace goodness) by not patching.
420 const GenericFnPtr LibcInfo::static_fn_[] = {
421 (GenericFnPtr)&::malloc,
422 (GenericFnPtr)&::free,
423 (GenericFnPtr)&::realloc,
424 (GenericFnPtr)&::calloc,
425 #ifdef __MINGW32__
426 NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
427 #else
428 (GenericFnPtr)(void*(*)(size_t))&::operator new,
429 (GenericFnPtr)(void*(*)(size_t))&::operator new[],
430 (GenericFnPtr)(void(*)(void*))&::operator delete,
431 (GenericFnPtr)(void(*)(void*))&::operator delete[],
432 (GenericFnPtr)
433 (void*(*)(size_t, struct std::nothrow_t const &))&::operator new,
434 (GenericFnPtr)
435 (void*(*)(size_t, struct std::nothrow_t const &))&::operator new[],
436 (GenericFnPtr)
437 (void(*)(void*, struct std::nothrow_t const &))&::operator delete,
438 (GenericFnPtr)
439 (void(*)(void*, struct std::nothrow_t const &))&::operator delete[],
440 #endif
441 (GenericFnPtr)&::_msize,
442 (GenericFnPtr)&::_expand,
443 (GenericFnPtr)&::calloc,
444 (GenericFnPtr)&::free,
445 (GenericFnPtr)&::free
446 };
447
448 template<int T> GenericFnPtr LibcInfoWithPatchFunctions<T>::origstub_fn_[] = {
449 // This will get filled in at run-time, as patching is done.
450 };
451
452 // _expand() is like realloc but doesn't move the
453 // pointer. We punt, which will cause callers to fall back on realloc.
454 static void* empty__expand(void*, size_t) __THROW {
455 return nullptr;
456 }
457
458 template<int T>
459 const GenericFnPtr LibcInfoWithPatchFunctions<T>::perftools_fn_[] = {
460 (GenericFnPtr)&Perftools_malloc,
461 (GenericFnPtr)&Perftools_free,
462 (GenericFnPtr)&Perftools_realloc,
463 (GenericFnPtr)&Perftools_calloc,
464 (GenericFnPtr)&Perftools_new,
465 (GenericFnPtr)&Perftools_newarray,
466 (GenericFnPtr)&Perftools_delete,
467 (GenericFnPtr)&Perftools_deletearray,
468 (GenericFnPtr)&Perftools_new_nothrow,
469 (GenericFnPtr)&Perftools_newarray_nothrow,
470 (GenericFnPtr)&Perftools_delete_nothrow,
471 (GenericFnPtr)&Perftools_deletearray_nothrow,
472 (GenericFnPtr)&Perftools__msize,
473 (GenericFnPtr)&empty__expand,
474 (GenericFnPtr)&Perftools_calloc,
475 (GenericFnPtr)&Perftools_free_base,
476 (GenericFnPtr)&Perftools_free_dbg
477 };
478
479 /*static*/ WindowsInfo::FunctionInfo WindowsInfo::function_info_[] = {
480 { "HeapAlloc", NULL, NULL, (GenericFnPtr)&Perftools_HeapAlloc },
481 { "HeapFree", NULL, NULL, (GenericFnPtr)&Perftools_HeapFree },
482 { "LoadLibraryExW", NULL, NULL, (GenericFnPtr)&Perftools_LoadLibraryExW },
483 { "FreeLibrary", NULL, NULL, (GenericFnPtr)&Perftools_FreeLibrary },
484 };
485
486 bool LibcInfo::PopulateWindowsFn(const ModuleEntryCopy& module_entry) {
487 // First, store the location of the function to patch before
488 // patching it. If none of these functions are found in the module,
489 // then this module has no libc in it, and we just return false.
490 for (int i = 0; i < kNumFunctions; i++) {
491 if (!function_name_[i]) // we can turn off patching by unsetting name
492 continue;
493 // The ::GetProcAddress calls were done in the ModuleEntryCopy
494 // constructor, so we don't have to make any windows calls here.
495 const GenericFnPtr fn = module_entry.rgProcAddresses[i];
496 if (fn) {
497 windows_fn_[i] = PreamblePatcher::ResolveTarget(fn);
498 }
499 }
500
501 // Some modules use the same function pointer for new and new[]. If
502 // we find that, set one of the pointers to NULL so we don't double-
503 // patch. Same may happen with new and nothrow-new, or even new[]
504 // and nothrow-new. It's easiest just to check each fn-ptr against
505 // every other.
506 for (int i = 0; i < kNumFunctions; i++) {
507 for (int j = i+1; j < kNumFunctions; j++) {
508 if (windows_fn_[i] == windows_fn_[j]) {
509 // We NULL the later one (j), so as to minimize the chances we
510 // NULL kFree and kRealloc. See comments below. This is fragile!
511 windows_fn_[j] = NULL;
512 }
513 }
514 }
515
516 // There's always a chance that our module uses the same function
517 // as another module that we've already loaded. In that case, we
518 // need to set our windows_fn to NULL, to avoid double-patching.
519 for (int ifn = 0; ifn < kNumFunctions; ifn++) {
520 for (int imod = 0;
521 imod < sizeof(g_module_libcs)/sizeof(*g_module_libcs); imod++) {
522 if (g_module_libcs[imod]->is_valid() &&
523 this->windows_fn(ifn) == g_module_libcs[imod]->windows_fn(ifn)) {
524 windows_fn_[ifn] = NULL;
525 }
526 }
527 }
528
529 bool found_non_null = false;
530 for (int i = 0; i < kNumFunctions; i++) {
531 if (windows_fn_[i])
532 found_non_null = true;
533 }
534 if (!found_non_null)
535 return false;
536
537 // It's important we didn't NULL out windows_fn_[kFree] or [kRealloc].
538 // The reason is, if those are NULL-ed out, we'll never patch them
539 // and thus never get an origstub_fn_ value for them, and when we
540 // try to call origstub_fn_[kFree/kRealloc] in Perftools_free and
541 // Perftools_realloc, below, it will fail. We could work around
542 // that by adding a pointer from one patch-unit to the other, but we
543 // haven't needed to yet.
544 CHECK(windows_fn_[kFree]);
545 CHECK(windows_fn_[kRealloc]);
546
547 // OK, we successfully populated. Let's store our member information.
548 module_base_address_ = module_entry.modBaseAddr;
549 module_base_size_ = module_entry.modBaseSize;
550 return true;
551 }
552
553 template<int T>
554 bool LibcInfoWithPatchFunctions<T>::Patch(const LibcInfo& me_info) {
555 CopyFrom(me_info); // copies the module_entry and the windows_fn_ array
556 for (int i = 0; i < kNumFunctions; i++) {
557 if (windows_fn_[i] && windows_fn_[i] != perftools_fn_[i]) {
558 // if origstub_fn_ is not NULL, it's left around from a previous
559 // patch. We need to set it to NULL for the new Patch call.
560 //
561 // Note that origstub_fn_ was logically freed by
562 // PreamblePatcher::Unpatch, so we don't have to do anything
563 // about it.
564 origstub_fn_[i] = NULL; // Patch() will fill this in
565 CHECK_EQ(sidestep::SIDESTEP_SUCCESS,
566 PreamblePatcher::Patch(windows_fn_[i], perftools_fn_[i],
567 &origstub_fn_[i]));
568 }
569 }
570 set_is_valid(true);
571 return true;
572 }
573
574 template<int T>
575 void LibcInfoWithPatchFunctions<T>::Unpatch() {
576 // We have to cast our GenericFnPtrs to void* for unpatch. This is
577 // contra the C++ spec; we use C-style casts to empahsize that.
578 for (int i = 0; i < kNumFunctions; i++) {
579 if (windows_fn_[i])
580 CHECK_EQ(sidestep::SIDESTEP_SUCCESS,
581 PreamblePatcher::Unpatch((void*)windows_fn_[i],
582 (void*)perftools_fn_[i],
583 (void*)origstub_fn_[i]));
584 }
585 set_is_valid(false);
586 }
587
588 void WindowsInfo::Patch() {
589 HMODULE hkernel32 = ::GetModuleHandleA("kernel32");
590 CHECK_NE(hkernel32, NULL);
591
592 // Unlike for libc, we know these exist in our module, so we can get
593 // and patch at the same time.
594 for (int i = 0; i < kNumFunctions; i++) {
595 function_info_[i].windows_fn = (GenericFnPtr)
596 ::GetProcAddress(hkernel32, function_info_[i].name);
597 // If origstub_fn is not NULL, it's left around from a previous
598 // patch. We need to set it to NULL for the new Patch call.
599 // Since we've patched Unpatch() not to delete origstub_fn_ (it
600 // causes problems in some contexts, though obviously not this
601 // one), we should delete it now, before setting it to NULL.
602 // NOTE: casting from a function to a pointer is contra the C++
603 // spec. It's not safe on IA64, but is on i386. We use
604 // a C-style cast here to emphasize this is not legal C++.
605 delete[] (char*)(function_info_[i].origstub_fn);
606 function_info_[i].origstub_fn = NULL; // Patch() will fill this in
607 CHECK_EQ(sidestep::SIDESTEP_SUCCESS,
608 PreamblePatcher::Patch(function_info_[i].windows_fn,
609 function_info_[i].perftools_fn,
610 &function_info_[i].origstub_fn));
611 }
612 }
613
614 void WindowsInfo::Unpatch() {
615 // We have to cast our GenericFnPtrs to void* for unpatch. This is
616 // contra the C++ spec; we use C-style casts to empahsize that.
617 for (int i = 0; i < kNumFunctions; i++) {
618 CHECK_EQ(sidestep::SIDESTEP_SUCCESS,
619 PreamblePatcher::Unpatch((void*)function_info_[i].windows_fn,
620 (void*)function_info_[i].perftools_fn,
621 (void*)function_info_[i].origstub_fn));
622 }
623 }
624
625 // You should hold the patch_all_modules_lock when calling this.
626 void PatchOneModuleLocked(const LibcInfo& me_info) {
627 // If we don't already have info on this module, let's add it. This
628 // is where we're sad that each libcX has a different type, so we
629 // can't use an array; instead, we have to use a switch statement.
630 // Patch() returns false if there were no libc functions in the module.
631 for (int i = 0; i < sizeof(g_module_libcs)/sizeof(*g_module_libcs); i++) {
632 if (!g_module_libcs[i]->is_valid()) { // found an empty spot to add!
633 switch (i) {
634 case 0: libc1.Patch(me_info); return;
635 case 1: libc2.Patch(me_info); return;
636 case 2: libc3.Patch(me_info); return;
637 case 3: libc4.Patch(me_info); return;
638 case 4: libc5.Patch(me_info); return;
639 case 5: libc6.Patch(me_info); return;
640 case 6: libc7.Patch(me_info); return;
641 case 7: libc8.Patch(me_info); return;
642 }
643 }
644 }
645 printf("PERFTOOLS ERROR: Too many modules containing libc in this executable\n");
646 }
647
648 void PatchMainExecutableLocked() {
649 if (main_executable.patched())
650 return; // main executable has already been patched
651 ModuleEntryCopy fake_module_entry; // make a fake one to pass into Patch()
652 // No need to call PopulateModuleEntryProcAddresses on the main executable.
653 main_executable.PopulateWindowsFn(fake_module_entry);
654 main_executable.Patch(main_executable);
655 }
656
657 // This lock is subject to a subtle and annoying lock inversion
658 // problem: it may interact badly with unknown internal windows locks.
659 // In particular, windows may be holding a lock when it calls
660 // LoadLibraryExW and FreeLibrary, which we've patched. We have those
661 // routines call PatchAllModules, which acquires this lock. If we
662 // make windows system calls while holding this lock, those system
663 // calls may need the internal windows locks that are being held in
664 // the call to LoadLibraryExW, resulting in deadlock. The solution is
665 // to be very careful not to call *any* windows routines while holding
666 // patch_all_modules_lock, inside PatchAllModules().
667 static SpinLock patch_all_modules_lock(SpinLock::LINKER_INITIALIZED);
668
669 // last_loaded: The set of modules that were loaded the last time
670 // PatchAllModules was called. This is an optimization for only
671 // looking at modules that were added or removed from the last call.
672 static std::set<HMODULE> *g_last_loaded;
673
674 // Iterates over all the modules currently loaded by the executable,
675 // according to windows, and makes sure they're all patched. Most
676 // modules will already be in loaded_modules, meaning we have already
677 // loaded and either patched them or determined they did not need to
678 // be patched. Others will not, which means we need to patch them
679 // (if necessary). Finally, we have to go through the existing
680 // g_module_libcs and see if any of those are *not* in the modules
681 // currently loaded by the executable. If so, we need to invalidate
682 // them. Returns true if we did any work (patching or invalidating),
683 // false if we were a noop. May update loaded_modules as well.
684 // NOTE: you must hold the patch_all_modules_lock to access loaded_modules.
685 bool PatchAllModules() {
686 std::vector<ModuleEntryCopy> modules;
687 bool made_changes = false;
688
689 const HANDLE hCurrentProcess = GetCurrentProcess();
690 DWORD num_modules = 0;
691 HMODULE hModules[kMaxModules]; // max # of modules we support in one process
692 if (!::EnumProcessModules(hCurrentProcess, hModules, sizeof(hModules),
693 &num_modules)) {
694 num_modules = 0;
695 }
696 // EnumProcessModules actually set the bytes written into hModules,
697 // so we need to divide to make num_modules actually be a module-count.
698 num_modules /= sizeof(*hModules);
699 if (num_modules >= kMaxModules) {
700 printf("PERFTOOLS ERROR: Too many modules in this executable to try"
701 " to patch them all (if you need to, raise kMaxModules in"
702 " patch_functions.cc).\n");
703 num_modules = kMaxModules;
704 }
705
706 // Now we handle the unpatching of modules we have in g_module_libcs
707 // but that were not found in EnumProcessModules. We need to
708 // invalidate them. To speed that up, we store the EnumProcessModules
709 // output in a set.
710 // At the same time, we prepare for the adding of new modules, by
711 // removing from hModules all the modules we know we've already
712 // patched (or decided don't need to be patched). At the end,
713 // hModules will hold only the modules that we need to consider patching.
714 std::set<HMODULE> currently_loaded_modules;
715 {
716 SpinLockHolder h(&patch_all_modules_lock);
717 if (!g_last_loaded) g_last_loaded = new std::set<HMODULE>;
718 // At the end of this loop, currently_loaded_modules contains the
719 // full list of EnumProcessModules, and hModules just the ones we
720 // haven't handled yet.
721 for (int i = 0; i < num_modules; ) {
722 currently_loaded_modules.insert(hModules[i]);
723 if (g_last_loaded->count(hModules[i]) > 0) {
724 hModules[i] = hModules[--num_modules]; // replace element i with tail
725 } else {
726 i++; // keep element i
727 }
728 }
729 // Now we do the unpatching/invalidation.
730 for (int i = 0; i < sizeof(g_module_libcs)/sizeof(*g_module_libcs); i++) {
731 if (g_module_libcs[i]->patched() &&
732 currently_loaded_modules.count(g_module_libcs[i]->hmodule()) == 0) {
733 // Means g_module_libcs[i] is no longer loaded (no me32 matched).
734 // We could call Unpatch() here, but why bother? The module
735 // has gone away, so nobody is going to call into it anyway.
736 g_module_libcs[i]->set_is_valid(false);
737 made_changes = true;
738 }
739 }
740 // Update the loaded module cache.
741 g_last_loaded->swap(currently_loaded_modules);
742 }
743
744 // Now that we know what modules are new, let's get the info we'll
745 // need to patch them. Note this *cannot* be done while holding the
746 // lock, since it needs to make windows calls (see the lock-inversion
747 // comments before the definition of patch_all_modules_lock).
748 MODULEINFO mi;
749 for (int i = 0; i < num_modules; i++) {
750 if (::GetModuleInformation(hCurrentProcess, hModules[i], &mi, sizeof(mi)))
751 modules.push_back(ModuleEntryCopy(mi));
752 }
753
754 // Now we can do the patching of new modules.
755 {
756 SpinLockHolder h(&patch_all_modules_lock);
757 for (std::vector<ModuleEntryCopy>::iterator it = modules.begin();
758 it != modules.end(); ++it) {
759 LibcInfo libc_info;
760 if (libc_info.PopulateWindowsFn(*it)) { // true==module has libc routines
761 PatchOneModuleLocked(libc_info);
762 made_changes = true;
763 }
764 }
765
766 // Now that we've dealt with the modules (dlls), update the main
767 // executable. We do this last because PatchMainExecutableLocked
768 // wants to look at how other modules were patched.
769 if (!main_executable.patched()) {
770 PatchMainExecutableLocked();
771 made_changes = true;
772 }
773 }
774 // TODO(csilvers): for this to be reliable, we need to also take
775 // into account if we *would* have patched any modules had they not
776 // already been loaded. (That is, made_changes should ignore
777 // g_last_loaded.)
778 return made_changes;
779 }
780
781
782 } // end unnamed namespace
783
784 // ---------------------------------------------------------------------
785 // Now that we've done all the patching machinery, let's actually
786 // define the functions we're patching in. Mostly these are
787 // simple wrappers around the do_* routines in tcmalloc.cc.
788 //
789 // In fact, we #include tcmalloc.cc to get at the tcmalloc internal
790 // do_* functions, the better to write our own hook functions.
791 // U-G-L-Y, I know. But the alternatives are, perhaps, worse. This
792 // also lets us define _msize(), _expand(), and other windows-specific
793 // functions here, using tcmalloc internals, without polluting
794 // tcmalloc.cc.
795 // -------------------------------------------------------------------
796
797 // TODO(csilvers): refactor tcmalloc.cc into two files, so I can link
798 // against the file with do_malloc, and ignore the one with malloc.
799 #include "tcmalloc.cc"
800
801 template<int T>
802 void* LibcInfoWithPatchFunctions<T>::Perftools_malloc(size_t size) __THROW {
803 return malloc_fast_path<tcmalloc::malloc_oom>(size);
804 }
805
806 template<int T>
807 void LibcInfoWithPatchFunctions<T>::Perftools_free(void* ptr) __THROW {
808 MallocHook::InvokeDeleteHook(ptr);
809 // This calls the windows free if do_free decides ptr was not
810 // allocated by tcmalloc. Note it calls the origstub_free from
811 // *this* templatized instance of LibcInfo. See "template
812 // trickiness" above.
813 do_free_with_callback(ptr, (void (*)(void*))origstub_fn_[kFree], false, 0);
814 }
815
816 template<int T>
817 void LibcInfoWithPatchFunctions<T>::Perftools_free_base(void* ptr) __THROW{
818 MallocHook::InvokeDeleteHook(ptr);
819 // This calls the windows free if do_free decides ptr was not
820 // allocated by tcmalloc. Note it calls the origstub_free from
821 // *this* templatized instance of LibcInfo. See "template
822 // trickiness" above.
823 do_free_with_callback(ptr, (void(*)(void*))origstub_fn_[kFreeBase], false, 0);
824 }
825
826 template<int T>
827 void LibcInfoWithPatchFunctions<T>::Perftools_free_dbg(void* ptr, int block_use) __THROW {
828 MallocHook::InvokeDeleteHook(ptr);
829 // The windows _free_dbg is called if ptr isn't owned by tcmalloc.
830 if (MallocExtension::instance()->GetOwnership(ptr) == MallocExtension::kOwned) {
831 do_free(ptr);
832 } else {
833 reinterpret_cast<void (*)(void*, int)>(origstub_fn_[kFreeDbg])(ptr, block_use);
834 }
835 }
836
837 template<int T>
838 void* LibcInfoWithPatchFunctions<T>::Perftools_realloc(
839 void* old_ptr, size_t new_size) __THROW {
840 if (old_ptr == NULL) {
841 void* result = do_malloc_or_cpp_alloc(new_size);
842 MallocHook::InvokeNewHook(result, new_size);
843 return result;
844 }
845 if (new_size == 0) {
846 MallocHook::InvokeDeleteHook(old_ptr);
847 do_free_with_callback(old_ptr,
848 (void (*)(void*))origstub_fn_[kFree], false, 0);
849 return NULL;
850 }
851 return do_realloc_with_callback(
852 old_ptr, new_size,
853 (void (*)(void*))origstub_fn_[kFree],
854 (size_t (*)(const void*))origstub_fn_[k_Msize]);
855 }
856
857 template<int T>
858 void* LibcInfoWithPatchFunctions<T>::Perftools_calloc(
859 size_t n, size_t elem_size) __THROW {
860 void* result = do_calloc(n, elem_size);
861 MallocHook::InvokeNewHook(result, n * elem_size);
862 return result;
863 }
864
865 template<int T>
866 void* LibcInfoWithPatchFunctions<T>::Perftools_new(size_t size) {
867 return malloc_fast_path<tcmalloc::cpp_throw_oom>(size);
868 }
869
870 template<int T>
871 void* LibcInfoWithPatchFunctions<T>::Perftools_newarray(size_t size) {
872 return malloc_fast_path<tcmalloc::cpp_throw_oom>(size);
873 }
874
875 template<int T>
876 void LibcInfoWithPatchFunctions<T>::Perftools_delete(void *p) {
877 MallocHook::InvokeDeleteHook(p);
878 do_free_with_callback(p, (void (*)(void*))origstub_fn_[kFree], false, 0);
879 }
880
881 template<int T>
882 void LibcInfoWithPatchFunctions<T>::Perftools_deletearray(void *p) {
883 MallocHook::InvokeDeleteHook(p);
884 do_free_with_callback(p, (void (*)(void*))origstub_fn_[kFree], false, 0);
885 }
886
887 template<int T>
888 void* LibcInfoWithPatchFunctions<T>::Perftools_new_nothrow(
889 size_t size, const std::nothrow_t&) __THROW {
890 return malloc_fast_path<tcmalloc::cpp_nothrow_oom>(size);
891 }
892
893 template<int T>
894 void* LibcInfoWithPatchFunctions<T>::Perftools_newarray_nothrow(
895 size_t size, const std::nothrow_t&) __THROW {
896 return malloc_fast_path<tcmalloc::cpp_nothrow_oom>(size);
897 }
898
899 template<int T>
900 void LibcInfoWithPatchFunctions<T>::Perftools_delete_nothrow(
901 void *p, const std::nothrow_t&) __THROW {
902 MallocHook::InvokeDeleteHook(p);
903 do_free_with_callback(p, (void (*)(void*))origstub_fn_[kFree], false, 0);
904 }
905
906 template<int T>
907 void LibcInfoWithPatchFunctions<T>::Perftools_deletearray_nothrow(
908 void *p, const std::nothrow_t&) __THROW {
909 MallocHook::InvokeDeleteHook(p);
910 do_free_with_callback(p, (void (*)(void*))origstub_fn_[kFree], false, 0);
911 }
912
913
914 // _msize() lets you figure out how much space is reserved for a
915 // pointer, in Windows. Even if applications don't call it, any DLL
916 // with global constructors will call (transitively) something called
917 // __dllonexit_lk in order to make sure the destructors get called
918 // when the dll unloads. And that will call msize -- horrible things
919 // can ensue if this is not hooked. Other parts of libc may also call
920 // this internally.
921
922 template<int T>
923 size_t LibcInfoWithPatchFunctions<T>::Perftools__msize(void* ptr) __THROW {
924 return GetSizeWithCallback(ptr, (size_t (*)(const void*))origstub_fn_[k_Msize]);
925 }
926
927 LPVOID WINAPI WindowsInfo::Perftools_HeapAlloc(HANDLE hHeap, DWORD dwFlags,
928 DWORD_PTR dwBytes) {
929 LPVOID result = ((LPVOID (WINAPI *)(HANDLE, DWORD, DWORD_PTR))
930 function_info_[kHeapAlloc].origstub_fn)(
931 hHeap, dwFlags, dwBytes);
932 MallocHook::InvokeNewHook(result, dwBytes);
933 return result;
934 }
935
936 BOOL WINAPI WindowsInfo::Perftools_HeapFree(HANDLE hHeap, DWORD dwFlags,
937 LPVOID lpMem) {
938 MallocHook::InvokeDeleteHook(lpMem);
939 return ((BOOL (WINAPI *)(HANDLE, DWORD, LPVOID))
940 function_info_[kHeapFree].origstub_fn)(
941 hHeap, dwFlags, lpMem);
942 }
943
944 HMODULE WINAPI WindowsInfo::Perftools_LoadLibraryExW(LPCWSTR lpFileName,
945 HANDLE hFile,
946 DWORD dwFlags) {
947 HMODULE rv;
948 // Check to see if the modules is already loaded, flag 0 gets a
949 // reference if it was loaded. If it was loaded no need to call
950 // PatchAllModules, just increase the reference count to match
951 // what GetModuleHandleExW does internally inside windows.
952 if (::GetModuleHandleExW(0, lpFileName, &rv)) {
953 return rv;
954 } else {
955 // Not already loaded, so load it.
956 rv = ((HMODULE (WINAPI *)(LPCWSTR, HANDLE, DWORD))
957 function_info_[kLoadLibraryExW].origstub_fn)(
958 lpFileName, hFile, dwFlags);
959 // This will patch any newly loaded libraries, if patching needs
960 // to be done.
961 PatchAllModules();
962
963 return rv;
964 }
965 }
966
967 BOOL WINAPI WindowsInfo::Perftools_FreeLibrary(HMODULE hLibModule) {
968 BOOL rv = ((BOOL (WINAPI *)(HMODULE))
969 function_info_[kFreeLibrary].origstub_fn)(hLibModule);
970
971 // Check to see if the module is still loaded by passing the base
972 // address and seeing if it comes back with the same address. If it
973 // is the same address it's still loaded, so the FreeLibrary() call
974 // was a noop, and there's no need to redo the patching.
975 HMODULE owner = NULL;
976 BOOL result = ::GetModuleHandleExW(
977 (GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
978 GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT),
979 (LPCWSTR)hLibModule,
980 &owner);
981 if (result && owner == hLibModule)
982 return rv;
983
984 PatchAllModules(); // this will fix up the list of patched libraries
985 return rv;
986 }
987
988
989 // ---------------------------------------------------------------------
990 // PatchWindowsFunctions()
991 // This is the function that is exposed to the outside world.
992 // It should be called before the program becomes multi-threaded,
993 // since main_executable_windows.Patch() is not thread-safe.
994 // ---------------------------------------------------------------------
995
996 void PatchWindowsFunctions() {
997 const char* disable_env = TCMallocGetenvSafe("TCMALLOC_DISABLE_REPLACEMENT");
998 const bool should_skip = disable_env &&
999 disable_env[0] == '1' &&
1000 disable_env[1] == '\0';
1001
1002 if (!should_skip) {
1003 // This does the libc patching in every module, and the main executable.
1004 PatchAllModules();
1005 main_executable_windows.Patch();
1006 }
1007 }
1008
1009 #if 0
1010 // It's possible to unpatch all the functions when we are exiting.
1011
1012 // The idea is to handle properly windows-internal data that is
1013 // allocated before PatchWindowsFunctions is called. If all
1014 // destruction happened in reverse order from construction, then we
1015 // could call UnpatchWindowsFunctions at just the right time, so that
1016 // that early-allocated data would be freed using the windows
1017 // allocation functions rather than tcmalloc. The problem is that
1018 // windows allocates some structures lazily, so it would allocate them
1019 // late (using tcmalloc) and then try to deallocate them late as well.
1020 // So instead of unpatching, we just modify all the tcmalloc routines
1021 // so they call through to the libc rountines if the memory in
1022 // question doesn't seem to have been allocated with tcmalloc. I keep
1023 // this unpatch code around for reference.
1024
1025 void UnpatchWindowsFunctions() {
1026 // We need to go back to the system malloc/etc at global destruct time,
1027 // so objects that were constructed before tcmalloc, using the system
1028 // malloc, can destroy themselves using the system free. This depends
1029 // on DLLs unloading in the reverse order in which they load!
1030 //
1031 // We also go back to the default HeapAlloc/etc, just for consistency.
1032 // Who knows, it may help avoid weird bugs in some situations.
1033 main_executable_windows.Unpatch();
1034 main_executable.Unpatch();
1035 if (libc1.is_valid()) libc1.Unpatch();
1036 if (libc2.is_valid()) libc2.Unpatch();
1037 if (libc3.is_valid()) libc3.Unpatch();
1038 if (libc4.is_valid()) libc4.Unpatch();
1039 if (libc5.is_valid()) libc5.Unpatch();
1040 if (libc6.is_valid()) libc6.Unpatch();
1041 if (libc7.is_valid()) libc7.Unpatch();
1042 if (libc8.is_valid()) libc8.Unpatch();
1043 }
1044 #endif