| 1 |
// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- |
| 2 |
// Copyright (c) 2007, Google Inc. |
| 3 |
// All rights reserved. |
| 4 |
// |
| 5 |
// Redistribution and use in source and binary forms, with or without |
| 6 |
// modification, are permitted provided that the following conditions are |
| 7 |
// met: |
| 8 |
// |
| 9 |
// * Redistributions of source code must retain the above copyright |
| 10 |
// notice, this list of conditions and the following disclaimer. |
| 11 |
// * Redistributions in binary form must reproduce the above |
| 12 |
// copyright notice, this list of conditions and the following disclaimer |
| 13 |
// in the documentation and/or other materials provided with the |
| 14 |
// distribution. |
| 15 |
// * Neither the name of Google Inc. nor the names of its |
| 16 |
// contributors may be used to endorse or promote products derived from |
| 17 |
// this software without specific prior written permission. |
| 18 |
// |
| 19 |
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 20 |
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 21 |
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 22 |
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 23 |
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 24 |
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 25 |
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 26 |
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 27 |
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 28 |
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 29 |
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 30 |
// |
| 31 |
// --- |
| 32 |
// Author: Craig Silverstein |
| 33 |
// |
| 34 |
// The main purpose of this file is to patch the libc allocation |
| 35 |
// routines (malloc and friends, but also _msize and other |
| 36 |
// windows-specific libc-style routines). However, we also patch |
| 37 |
// windows routines to do accounting. We do better at the former than |
| 38 |
// the latter. Here are some comments from Paul Pluzhnikov about what |
| 39 |
// it might take to do a really good job patching windows routines to |
| 40 |
// keep track of memory usage: |
| 41 |
// |
| 42 |
// "You should intercept at least the following: |
| 43 |
// HeapCreate HeapDestroy HeapAlloc HeapReAlloc HeapFree |
| 44 |
// RtlCreateHeap RtlDestroyHeap RtlAllocateHeap RtlFreeHeap |
| 45 |
// malloc calloc realloc free |
| 46 |
// malloc_dbg calloc_dbg realloc_dbg free_dbg |
| 47 |
// Some of these call the other ones (but not always), sometimes |
| 48 |
// recursively (i.e. HeapCreate may call HeapAlloc on a different |
| 49 |
// heap, IIRC)." |
| 50 |
// |
| 51 |
// Since Paul didn't mention VirtualAllocEx, he may not have even been |
| 52 |
// considering all the mmap-like functions that windows has (or he may |
| 53 |
// just be ignoring it because he's seen we already patch it). Of the |
| 54 |
// above, we do not patch the *_dbg functions, and of the windows |
| 55 |
// functions, we only patch HeapAlloc and HeapFree. |
| 56 |
// |
| 57 |
// The *_dbg functions come into play with /MDd, /MTd, and /MLd, |
| 58 |
// probably. It may be ok to just turn off tcmalloc in those cases -- |
| 59 |
// if the user wants the windows debug malloc, they probably don't |
| 60 |
// want tcmalloc! We should also test with all of /MD, /MT, and /ML, |
| 61 |
// which we're not currently doing. |
| 62 |
|
| 63 |
// TODO(csilvers): try to do better here? Paul does conclude: |
| 64 |
// "Keeping track of all of this was a nightmare." |
| 65 |
|
| 66 |
#ifndef _WIN32 |
| 67 |
# error You should only be including windows/patch_functions.cc in a windows environment! |
| 68 |
#endif |
| 69 |
|
| 70 |
#include <config.h> |
| 71 |
|
| 72 |
#ifdef WIN32_OVERRIDE_ALLOCATORS |
| 73 |
#error This file is intended for patching allocators - use override_functions.cc instead. |
| 74 |
#endif |
| 75 |
|
| 76 |
// We use psapi. Non-MSVC systems will have to link this in themselves. |
| 77 |
#ifdef _MSC_VER |
| 78 |
#pragma comment(lib, "Psapi.lib") |
| 79 |
#endif |
| 80 |
|
| 81 |
// Make sure we always use the 'old' names of the psapi functions. |
| 82 |
#ifndef PSAPI_VERSION |
| 83 |
#define PSAPI_VERSION 1 |
| 84 |
#endif |
| 85 |
|
| 86 |
#include <windows.h> |
| 87 |
#include <stdio.h> |
| 88 |
#include <malloc.h> // for _msize and _expand |
| 89 |
#include <psapi.h> // for EnumProcessModules, GetModuleInformation, etc. |
| 90 |
#include <set> |
| 91 |
#include <map> |
| 92 |
#include <vector> |
| 93 |
#include <base/logging.h> |
| 94 |
#include "base/spinlock.h" |
| 95 |
#include "getenv_safe.h" // for TCMallocGetenvSafe |
| 96 |
#include "gperftools/malloc_hook.h" |
| 97 |
#include "malloc_hook-inl.h" |
| 98 |
#include "preamble_patcher.h" |
| 99 |
|
| 100 |
// The maximum number of modules we allow to be in one executable |
| 101 |
const int kMaxModules = 8182; |
| 102 |
|
| 103 |
// These are hard-coded, unfortunately. :-( They are also probably |
| 104 |
// compiler specific. See get_mangled_names.cc, in this directory, |
| 105 |
// for instructions on how to update these names for your compiler. |
| 106 |
#ifdef _WIN64 |
| 107 |
const char kMangledNew[] = "??2@YAPEAX_K@Z"; |
| 108 |
const char kMangledNewArray[] = "??_U@YAPEAX_K@Z"; |
| 109 |
const char kMangledDelete[] = "??3@YAXPEAX@Z"; |
| 110 |
const char kMangledDeleteArray[] = "??_V@YAXPEAX@Z"; |
| 111 |
const char kMangledNewNothrow[] = "??2@YAPEAX_KAEBUnothrow_t@std@@@Z"; |
| 112 |
const char kMangledNewArrayNothrow[] = "??_U@YAPEAX_KAEBUnothrow_t@std@@@Z"; |
| 113 |
const char kMangledDeleteNothrow[] = "??3@YAXPEAXAEBUnothrow_t@std@@@Z"; |
| 114 |
const char kMangledDeleteArrayNothrow[] = "??_V@YAXPEAXAEBUnothrow_t@std@@@Z"; |
| 115 |
#else |
| 116 |
const char kMangledNew[] = "??2@YAPAXI@Z"; |
| 117 |
const char kMangledNewArray[] = "??_U@YAPAXI@Z"; |
| 118 |
const char kMangledDelete[] = "??3@YAXPAX@Z"; |
| 119 |
const char kMangledDeleteArray[] = "??_V@YAXPAX@Z"; |
| 120 |
const char kMangledNewNothrow[] = "??2@YAPAXIABUnothrow_t@std@@@Z"; |
| 121 |
const char kMangledNewArrayNothrow[] = "??_U@YAPAXIABUnothrow_t@std@@@Z"; |
| 122 |
const char kMangledDeleteNothrow[] = "??3@YAXPAXABUnothrow_t@std@@@Z"; |
| 123 |
const char kMangledDeleteArrayNothrow[] = "??_V@YAXPAXABUnothrow_t@std@@@Z"; |
| 124 |
#endif |
| 125 |
|
| 126 |
// This is an unused but exported symbol that we can use to tell the |
| 127 |
// MSVC linker to bring in libtcmalloc, via the /INCLUDE linker flag. |
| 128 |
// Without this, the linker will likely decide that libtcmalloc.dll |
| 129 |
// doesn't add anything to the executable (since it does all its work |
| 130 |
// through patching, which the linker can't see), and ignore it |
| 131 |
// entirely. (The name 'tcmalloc' is already reserved for a |
| 132 |
// namespace. I'd rather export a variable named "_tcmalloc", but I |
| 133 |
// couldn't figure out how to get that to work. This function exports |
| 134 |
// the symbol "__tcmalloc".) |
| 135 |
extern "C" PERFTOOLS_DLL_DECL void _tcmalloc(); |
| 136 |
void _tcmalloc() { } |
| 137 |
|
| 138 |
// This is the version needed for windows x64, which has a different |
| 139 |
// decoration scheme which doesn't auto-add a leading underscore. |
| 140 |
extern "C" PERFTOOLS_DLL_DECL void __tcmalloc(); |
| 141 |
void __tcmalloc() { } |
| 142 |
|
| 143 |
namespace { // most everything here is in an unnamed namespace |
| 144 |
|
| 145 |
typedef void (*GenericFnPtr)(); |
| 146 |
|
| 147 |
using sidestep::PreamblePatcher; |
| 148 |
|
| 149 |
struct ModuleEntryCopy; // defined below |
| 150 |
|
| 151 |
// These functions are how we override the memory allocation |
| 152 |
// functions, just like tcmalloc.cc and malloc_hook.cc do. |
| 153 |
|
| 154 |
// This is information about the routines we're patching, for a given |
| 155 |
// module that implements libc memory routines. A single executable |
| 156 |
// can have several libc implementations running about (in different |
| 157 |
// .dll's), and we need to patch/unpatch them all. This defines |
| 158 |
// everything except the new functions we're patching in, which |
| 159 |
// are defined in LibcFunctions, below. |
| 160 |
class LibcInfo { |
| 161 |
public: |
| 162 |
LibcInfo() { |
| 163 |
memset(this, 0, sizeof(*this)); // easiest way to initialize the array |
| 164 |
} |
| 165 |
|
| 166 |
bool patched() const { return is_valid(); } |
| 167 |
void set_is_valid(bool b) { is_valid_ = b; } |
| 168 |
// According to http://msdn.microsoft.com/en-us/library/ms684229(VS.85).aspx: |
| 169 |
// "The load address of a module (lpBaseOfDll) is the same as the HMODULE |
| 170 |
// value." |
| 171 |
HMODULE hmodule() const { |
| 172 |
return reinterpret_cast<HMODULE>(const_cast<void*>(module_base_address_)); |
| 173 |
} |
| 174 |
|
| 175 |
// Populates all the windows_fn_[] vars based on our module info. |
| 176 |
// Returns false if windows_fn_ is all NULL's, because there's |
| 177 |
// nothing to patch. Also populates the rest of the module_entry |
| 178 |
// info, such as the module's name. |
| 179 |
bool PopulateWindowsFn(const ModuleEntryCopy& module_entry); |
| 180 |
|
| 181 |
protected: |
| 182 |
void CopyFrom(const LibcInfo& that) { |
| 183 |
if (this == &that) |
| 184 |
return; |
| 185 |
this->is_valid_ = that.is_valid_; |
| 186 |
memcpy(this->windows_fn_, that.windows_fn_, sizeof(windows_fn_)); |
| 187 |
this->module_base_address_ = that.module_base_address_; |
| 188 |
this->module_base_size_ = that.module_base_size_; |
| 189 |
} |
| 190 |
|
| 191 |
enum { |
| 192 |
kMalloc, kFree, kRealloc, kCalloc, |
| 193 |
kNew, kNewArray, kDelete, kDeleteArray, |
| 194 |
kNewNothrow, kNewArrayNothrow, kDeleteNothrow, kDeleteArrayNothrow, |
| 195 |
// These are windows-only functions from malloc.h |
| 196 |
k_Msize, k_Expand, |
| 197 |
// A MS CRT "internal" function, implemented using _calloc_impl |
| 198 |
k_CallocCrt, |
| 199 |
// Underlying deallocation functions called by CRT internal functions or operator delete |
| 200 |
kFreeBase, kFreeDbg, |
| 201 |
kNumFunctions |
| 202 |
}; |
| 203 |
|
| 204 |
// I'd like to put these together in a struct (perhaps in the |
| 205 |
// subclass, so we can put in perftools_fn_ as well), but vc8 seems |
| 206 |
// to have a bug where it doesn't initialize the struct properly if |
| 207 |
// we try to take the address of a function that's not yet loaded |
| 208 |
// from a dll, as is the common case for static_fn_. So we need |
| 209 |
// each to be in its own array. :-( |
| 210 |
static const char* const function_name_[kNumFunctions]; |
| 211 |
|
| 212 |
// This function is only used when statically linking the binary. |
| 213 |
// In that case, loading malloc/etc from the dll (via |
| 214 |
// PatchOneModule) won't work, since there are no dlls. Instead, |
| 215 |
// you just want to be taking the address of malloc/etc directly. |
| 216 |
// In the common, non-static-link case, these pointers will all be |
| 217 |
// NULL, since this initializer runs before msvcrt.dll is loaded. |
| 218 |
static const GenericFnPtr static_fn_[kNumFunctions]; |
| 219 |
|
| 220 |
// This is the address of the function we are going to patch |
| 221 |
// (malloc, etc). Other info about the function is in the |
| 222 |
// patch-specific subclasses, below. |
| 223 |
GenericFnPtr windows_fn_[kNumFunctions]; |
| 224 |
|
| 225 |
// This is set to true when this structure is initialized (because |
| 226 |
// we're patching a new library) and set to false when it's |
| 227 |
// uninitialized (because we've freed that library). |
| 228 |
bool is_valid_; |
| 229 |
|
| 230 |
const void *module_base_address_; |
| 231 |
size_t module_base_size_; |
| 232 |
|
| 233 |
public: |
| 234 |
// These shouldn't have to be public, since only subclasses of |
| 235 |
// LibcInfo need it, but they do. Maybe something to do with |
| 236 |
// templates. Shrug. I hide them down here so users won't see |
| 237 |
// them. :-) (OK, I also need to define ctrgProcAddress late.) |
| 238 |
bool is_valid() const { return is_valid_; } |
| 239 |
GenericFnPtr windows_fn(int ifunction) const { |
| 240 |
return windows_fn_[ifunction]; |
| 241 |
} |
| 242 |
// These three are needed by ModuleEntryCopy. |
| 243 |
static const int ctrgProcAddress = kNumFunctions; |
| 244 |
static GenericFnPtr static_fn(int ifunction) { |
| 245 |
return static_fn_[ifunction]; |
| 246 |
} |
| 247 |
static const char* const function_name(int ifunction) { |
| 248 |
return function_name_[ifunction]; |
| 249 |
} |
| 250 |
}; |
| 251 |
|
| 252 |
// Template trickiness: logically, a LibcInfo would include |
| 253 |
// Windows_malloc_, origstub_malloc_, and Perftools_malloc_: for a |
| 254 |
// given module, these three go together. And in fact, |
| 255 |
// Perftools_malloc_ may need to call origstub_malloc_, which means we |
| 256 |
// either need to change Perftools_malloc_ to take origstub_malloc_ as |
| 257 |
// an argument -- unfortunately impossible since it needs to keep the |
| 258 |
// same API as normal malloc -- or we need to write a different |
| 259 |
// version of Perftools_malloc_ for each LibcInfo instance we create. |
| 260 |
// We choose the second route, and use templates to implement it (we |
| 261 |
// could have also used macros). So to get multiple versions |
| 262 |
// of the struct, we say "struct<1> var1; struct<2> var2;". The price |
| 263 |
// we pay is some code duplication, and more annoying, each instance |
| 264 |
// of this var is a separate type. |
| 265 |
template<int> class LibcInfoWithPatchFunctions : public LibcInfo { |
| 266 |
public: |
| 267 |
// me_info should have had PopulateWindowsFn() called on it, so the |
| 268 |
// module_* vars and windows_fn_ are set up. |
| 269 |
bool Patch(const LibcInfo& me_info); |
| 270 |
void Unpatch(); |
| 271 |
|
| 272 |
private: |
| 273 |
// This holds the original function contents after we patch the function. |
| 274 |
// This has to be defined static in the subclass, because the perftools_fns |
| 275 |
// reference origstub_fn_. |
| 276 |
static GenericFnPtr origstub_fn_[kNumFunctions]; |
| 277 |
|
| 278 |
// This is the function we want to patch in |
| 279 |
static const GenericFnPtr perftools_fn_[kNumFunctions]; |
| 280 |
|
| 281 |
static void* Perftools_malloc(size_t size) __THROW; |
| 282 |
static void Perftools_free(void* ptr) __THROW; |
| 283 |
static void Perftools_free_base(void* ptr) __THROW; |
| 284 |
static void Perftools_free_dbg(void* ptr, int block_use) __THROW; |
| 285 |
static void* Perftools_realloc(void* ptr, size_t size) __THROW; |
| 286 |
static void* Perftools_calloc(size_t nmemb, size_t size) __THROW; |
| 287 |
static void* Perftools_new(size_t size); |
| 288 |
static void* Perftools_newarray(size_t size); |
| 289 |
static void Perftools_delete(void *ptr); |
| 290 |
static void Perftools_deletearray(void *ptr); |
| 291 |
static void* Perftools_new_nothrow(size_t size, |
| 292 |
const std::nothrow_t&) __THROW; |
| 293 |
static void* Perftools_newarray_nothrow(size_t size, |
| 294 |
const std::nothrow_t&) __THROW; |
| 295 |
static void Perftools_delete_nothrow(void *ptr, |
| 296 |
const std::nothrow_t&) __THROW; |
| 297 |
static void Perftools_deletearray_nothrow(void *ptr, |
| 298 |
const std::nothrow_t&) __THROW; |
| 299 |
static size_t Perftools__msize(void *ptr) __THROW; |
| 300 |
static void* Perftools__expand(void *ptr, size_t size) __THROW; |
| 301 |
// malloc.h also defines these functions: |
| 302 |
// _aligned_malloc, _aligned_free, |
| 303 |
// _recalloc, _aligned_offset_malloc, _aligned_realloc, _aligned_recalloc |
| 304 |
// _aligned_offset_realloc, _aligned_offset_recalloc, _malloca, _freea |
| 305 |
// But they seem pretty obscure, and I'm fine not overriding them for now. |
| 306 |
// It may be they all call into malloc/free anyway. |
| 307 |
}; |
| 308 |
|
| 309 |
// This is a subset of MODDULEENTRY32, that we need for patching. |
| 310 |
struct ModuleEntryCopy { |
| 311 |
LPVOID modBaseAddr; // the same as hmodule |
| 312 |
DWORD modBaseSize; |
| 313 |
// This is not part of MODDULEENTRY32, but is needed to avoid making |
| 314 |
// windows syscalls while we're holding patch_all_modules_lock (see |
| 315 |
// lock-inversion comments at patch_all_modules_lock definition, below). |
| 316 |
GenericFnPtr rgProcAddresses[LibcInfo::ctrgProcAddress]; |
| 317 |
|
| 318 |
ModuleEntryCopy() { |
| 319 |
modBaseAddr = NULL; |
| 320 |
modBaseSize = 0; |
| 321 |
for (int i = 0; i < sizeof(rgProcAddresses)/sizeof(*rgProcAddresses); i++) |
| 322 |
rgProcAddresses[i] = LibcInfo::static_fn(i); |
| 323 |
} |
| 324 |
ModuleEntryCopy(const MODULEINFO& mi) { |
| 325 |
this->modBaseAddr = mi.lpBaseOfDll; |
| 326 |
this->modBaseSize = mi.SizeOfImage; |
| 327 |
LPVOID modEndAddr = (char*)mi.lpBaseOfDll + mi.SizeOfImage; |
| 328 |
for (int i = 0; i < sizeof(rgProcAddresses)/sizeof(*rgProcAddresses); i++) { |
| 329 |
FARPROC target = ::GetProcAddress( |
| 330 |
reinterpret_cast<const HMODULE>(mi.lpBaseOfDll), |
| 331 |
LibcInfo::function_name(i)); |
| 332 |
void* target_addr = reinterpret_cast<void*>(target); |
| 333 |
// Sometimes a DLL forwards a function to a function in another |
| 334 |
// DLL. We don't want to patch those forwarded functions -- |
| 335 |
// they'll get patched when the other DLL is processed. |
| 336 |
if (modBaseAddr <= target_addr && target_addr < modEndAddr) |
| 337 |
rgProcAddresses[i] = (GenericFnPtr)target; |
| 338 |
else |
| 339 |
rgProcAddresses[i] = (GenericFnPtr)NULL; |
| 340 |
} |
| 341 |
} |
| 342 |
}; |
| 343 |
|
| 344 |
// This class is easier because there's only one of them. |
| 345 |
class WindowsInfo { |
| 346 |
public: |
| 347 |
void Patch(); |
| 348 |
void Unpatch(); |
| 349 |
|
| 350 |
private: |
| 351 |
// TODO(csilvers): should we be patching GlobalAlloc/LocalAlloc instead, |
| 352 |
// for pre-XP systems? |
| 353 |
enum { |
| 354 |
kHeapAlloc, kHeapFree, kLoadLibraryExW, kFreeLibrary, |
| 355 |
kNumFunctions |
| 356 |
}; |
| 357 |
|
| 358 |
struct FunctionInfo { |
| 359 |
const char* const name; // name of fn in a module (eg "malloc") |
| 360 |
GenericFnPtr windows_fn; // the fn whose name we call (&malloc) |
| 361 |
GenericFnPtr origstub_fn; // original fn contents after we patch |
| 362 |
const GenericFnPtr perftools_fn; // fn we want to patch in |
| 363 |
}; |
| 364 |
|
| 365 |
static FunctionInfo function_info_[kNumFunctions]; |
| 366 |
|
| 367 |
// A Windows-API equivalent of malloc and free |
| 368 |
static LPVOID WINAPI Perftools_HeapAlloc(HANDLE hHeap, DWORD dwFlags, |
| 369 |
DWORD_PTR dwBytes); |
| 370 |
static BOOL WINAPI Perftools_HeapFree(HANDLE hHeap, DWORD dwFlags, |
| 371 |
LPVOID lpMem); |
| 372 |
// We don't need the other 3 variants because they all call this one. */ |
| 373 |
static HMODULE WINAPI Perftools_LoadLibraryExW(LPCWSTR lpFileName, |
| 374 |
HANDLE hFile, |
| 375 |
DWORD dwFlags); |
| 376 |
static BOOL WINAPI Perftools_FreeLibrary(HMODULE hLibModule); |
| 377 |
}; |
| 378 |
|
| 379 |
// If you run out, just add a few more to the array. You'll also need |
| 380 |
// to update the switch statement in PatchOneModule(), and the list in |
| 381 |
// UnpatchWindowsFunctions(). |
| 382 |
// main_executable and main_executable_windows are two windows into |
| 383 |
// the same executable. One is responsible for patching the libc |
| 384 |
// routines that live in the main executable (if any) to use tcmalloc; |
| 385 |
// the other is responsible for patching the windows routines like |
| 386 |
// HeapAlloc/etc to use tcmalloc. |
| 387 |
static LibcInfoWithPatchFunctions<0> main_executable; |
| 388 |
static LibcInfoWithPatchFunctions<1> libc1; |
| 389 |
static LibcInfoWithPatchFunctions<2> libc2; |
| 390 |
static LibcInfoWithPatchFunctions<3> libc3; |
| 391 |
static LibcInfoWithPatchFunctions<4> libc4; |
| 392 |
static LibcInfoWithPatchFunctions<5> libc5; |
| 393 |
static LibcInfoWithPatchFunctions<6> libc6; |
| 394 |
static LibcInfoWithPatchFunctions<7> libc7; |
| 395 |
static LibcInfoWithPatchFunctions<8> libc8; |
| 396 |
static LibcInfo* g_module_libcs[] = { |
| 397 |
&libc1, &libc2, &libc3, &libc4, &libc5, &libc6, &libc7, &libc8 |
| 398 |
}; |
| 399 |
static WindowsInfo main_executable_windows; |
| 400 |
|
| 401 |
const char* const LibcInfo::function_name_[] = { |
| 402 |
"malloc", "free", "realloc", "calloc", |
| 403 |
kMangledNew, kMangledNewArray, kMangledDelete, kMangledDeleteArray, |
| 404 |
// Ideally we should patch the nothrow versions of new/delete, but |
| 405 |
// at least in msvcrt, nothrow-new machine-code is of a type we |
| 406 |
// can't patch. Since these are relatively rare, I'm hoping it's ok |
| 407 |
// not to patch them. (NULL name turns off patching.) |
| 408 |
NULL, // kMangledNewNothrow, |
| 409 |
NULL, // kMangledNewArrayNothrow, |
| 410 |
NULL, // kMangledDeleteNothrow, |
| 411 |
NULL, // kMangledDeleteArrayNothrow, |
| 412 |
"_msize", "_expand", "_calloc_crt", "_free_base", "_free_dbg" |
| 413 |
}; |
| 414 |
|
| 415 |
// For mingw, I can't patch the new/delete here, because the |
| 416 |
// instructions are too small to patch. Luckily, they're so small |
| 417 |
// because all they do is call into malloc/free, so they still end up |
| 418 |
// calling tcmalloc routines, and we don't actually lose anything |
| 419 |
// (except maybe some stacktrace goodness) by not patching. |
| 420 |
const GenericFnPtr LibcInfo::static_fn_[] = { |
| 421 |
(GenericFnPtr)&::malloc, |
| 422 |
(GenericFnPtr)&::free, |
| 423 |
(GenericFnPtr)&::realloc, |
| 424 |
(GenericFnPtr)&::calloc, |
| 425 |
#ifdef __MINGW32__ |
| 426 |
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, |
| 427 |
#else |
| 428 |
(GenericFnPtr)(void*(*)(size_t))&::operator new, |
| 429 |
(GenericFnPtr)(void*(*)(size_t))&::operator new[], |
| 430 |
(GenericFnPtr)(void(*)(void*))&::operator delete, |
| 431 |
(GenericFnPtr)(void(*)(void*))&::operator delete[], |
| 432 |
(GenericFnPtr) |
| 433 |
(void*(*)(size_t, struct std::nothrow_t const &))&::operator new, |
| 434 |
(GenericFnPtr) |
| 435 |
(void*(*)(size_t, struct std::nothrow_t const &))&::operator new[], |
| 436 |
(GenericFnPtr) |
| 437 |
(void(*)(void*, struct std::nothrow_t const &))&::operator delete, |
| 438 |
(GenericFnPtr) |
| 439 |
(void(*)(void*, struct std::nothrow_t const &))&::operator delete[], |
| 440 |
#endif |
| 441 |
(GenericFnPtr)&::_msize, |
| 442 |
(GenericFnPtr)&::_expand, |
| 443 |
(GenericFnPtr)&::calloc, |
| 444 |
(GenericFnPtr)&::free, |
| 445 |
(GenericFnPtr)&::free |
| 446 |
}; |
| 447 |
|
| 448 |
template<int T> GenericFnPtr LibcInfoWithPatchFunctions<T>::origstub_fn_[] = { |
| 449 |
// This will get filled in at run-time, as patching is done. |
| 450 |
}; |
| 451 |
|
| 452 |
// _expand() is like realloc but doesn't move the |
| 453 |
// pointer. We punt, which will cause callers to fall back on realloc. |
| 454 |
static void* empty__expand(void*, size_t) __THROW { |
| 455 |
return nullptr; |
| 456 |
} |
| 457 |
|
| 458 |
template<int T> |
| 459 |
const GenericFnPtr LibcInfoWithPatchFunctions<T>::perftools_fn_[] = { |
| 460 |
(GenericFnPtr)&Perftools_malloc, |
| 461 |
(GenericFnPtr)&Perftools_free, |
| 462 |
(GenericFnPtr)&Perftools_realloc, |
| 463 |
(GenericFnPtr)&Perftools_calloc, |
| 464 |
(GenericFnPtr)&Perftools_new, |
| 465 |
(GenericFnPtr)&Perftools_newarray, |
| 466 |
(GenericFnPtr)&Perftools_delete, |
| 467 |
(GenericFnPtr)&Perftools_deletearray, |
| 468 |
(GenericFnPtr)&Perftools_new_nothrow, |
| 469 |
(GenericFnPtr)&Perftools_newarray_nothrow, |
| 470 |
(GenericFnPtr)&Perftools_delete_nothrow, |
| 471 |
(GenericFnPtr)&Perftools_deletearray_nothrow, |
| 472 |
(GenericFnPtr)&Perftools__msize, |
| 473 |
(GenericFnPtr)&empty__expand, |
| 474 |
(GenericFnPtr)&Perftools_calloc, |
| 475 |
(GenericFnPtr)&Perftools_free_base, |
| 476 |
(GenericFnPtr)&Perftools_free_dbg |
| 477 |
}; |
| 478 |
|
| 479 |
/*static*/ WindowsInfo::FunctionInfo WindowsInfo::function_info_[] = { |
| 480 |
{ "HeapAlloc", NULL, NULL, (GenericFnPtr)&Perftools_HeapAlloc }, |
| 481 |
{ "HeapFree", NULL, NULL, (GenericFnPtr)&Perftools_HeapFree }, |
| 482 |
{ "LoadLibraryExW", NULL, NULL, (GenericFnPtr)&Perftools_LoadLibraryExW }, |
| 483 |
{ "FreeLibrary", NULL, NULL, (GenericFnPtr)&Perftools_FreeLibrary }, |
| 484 |
}; |
| 485 |
|
| 486 |
bool LibcInfo::PopulateWindowsFn(const ModuleEntryCopy& module_entry) { |
| 487 |
// First, store the location of the function to patch before |
| 488 |
// patching it. If none of these functions are found in the module, |
| 489 |
// then this module has no libc in it, and we just return false. |
| 490 |
for (int i = 0; i < kNumFunctions; i++) { |
| 491 |
if (!function_name_[i]) // we can turn off patching by unsetting name |
| 492 |
continue; |
| 493 |
// The ::GetProcAddress calls were done in the ModuleEntryCopy |
| 494 |
// constructor, so we don't have to make any windows calls here. |
| 495 |
const GenericFnPtr fn = module_entry.rgProcAddresses[i]; |
| 496 |
if (fn) { |
| 497 |
windows_fn_[i] = PreamblePatcher::ResolveTarget(fn); |
| 498 |
} |
| 499 |
} |
| 500 |
|
| 501 |
// Some modules use the same function pointer for new and new[]. If |
| 502 |
// we find that, set one of the pointers to NULL so we don't double- |
| 503 |
// patch. Same may happen with new and nothrow-new, or even new[] |
| 504 |
// and nothrow-new. It's easiest just to check each fn-ptr against |
| 505 |
// every other. |
| 506 |
for (int i = 0; i < kNumFunctions; i++) { |
| 507 |
for (int j = i+1; j < kNumFunctions; j++) { |
| 508 |
if (windows_fn_[i] == windows_fn_[j]) { |
| 509 |
// We NULL the later one (j), so as to minimize the chances we |
| 510 |
// NULL kFree and kRealloc. See comments below. This is fragile! |
| 511 |
windows_fn_[j] = NULL; |
| 512 |
} |
| 513 |
} |
| 514 |
} |
| 515 |
|
| 516 |
// There's always a chance that our module uses the same function |
| 517 |
// as another module that we've already loaded. In that case, we |
| 518 |
// need to set our windows_fn to NULL, to avoid double-patching. |
| 519 |
for (int ifn = 0; ifn < kNumFunctions; ifn++) { |
| 520 |
for (int imod = 0; |
| 521 |
imod < sizeof(g_module_libcs)/sizeof(*g_module_libcs); imod++) { |
| 522 |
if (g_module_libcs[imod]->is_valid() && |
| 523 |
this->windows_fn(ifn) == g_module_libcs[imod]->windows_fn(ifn)) { |
| 524 |
windows_fn_[ifn] = NULL; |
| 525 |
} |
| 526 |
} |
| 527 |
} |
| 528 |
|
| 529 |
bool found_non_null = false; |
| 530 |
for (int i = 0; i < kNumFunctions; i++) { |
| 531 |
if (windows_fn_[i]) |
| 532 |
found_non_null = true; |
| 533 |
} |
| 534 |
if (!found_non_null) |
| 535 |
return false; |
| 536 |
|
| 537 |
// It's important we didn't NULL out windows_fn_[kFree] or [kRealloc]. |
| 538 |
// The reason is, if those are NULL-ed out, we'll never patch them |
| 539 |
// and thus never get an origstub_fn_ value for them, and when we |
| 540 |
// try to call origstub_fn_[kFree/kRealloc] in Perftools_free and |
| 541 |
// Perftools_realloc, below, it will fail. We could work around |
| 542 |
// that by adding a pointer from one patch-unit to the other, but we |
| 543 |
// haven't needed to yet. |
| 544 |
CHECK(windows_fn_[kFree]); |
| 545 |
CHECK(windows_fn_[kRealloc]); |
| 546 |
|
| 547 |
// OK, we successfully populated. Let's store our member information. |
| 548 |
module_base_address_ = module_entry.modBaseAddr; |
| 549 |
module_base_size_ = module_entry.modBaseSize; |
| 550 |
return true; |
| 551 |
} |
| 552 |
|
| 553 |
template<int T> |
| 554 |
bool LibcInfoWithPatchFunctions<T>::Patch(const LibcInfo& me_info) { |
| 555 |
CopyFrom(me_info); // copies the module_entry and the windows_fn_ array |
| 556 |
for (int i = 0; i < kNumFunctions; i++) { |
| 557 |
if (windows_fn_[i] && windows_fn_[i] != perftools_fn_[i]) { |
| 558 |
// if origstub_fn_ is not NULL, it's left around from a previous |
| 559 |
// patch. We need to set it to NULL for the new Patch call. |
| 560 |
// |
| 561 |
// Note that origstub_fn_ was logically freed by |
| 562 |
// PreamblePatcher::Unpatch, so we don't have to do anything |
| 563 |
// about it. |
| 564 |
origstub_fn_[i] = NULL; // Patch() will fill this in |
| 565 |
CHECK_EQ(sidestep::SIDESTEP_SUCCESS, |
| 566 |
PreamblePatcher::Patch(windows_fn_[i], perftools_fn_[i], |
| 567 |
&origstub_fn_[i])); |
| 568 |
} |
| 569 |
} |
| 570 |
set_is_valid(true); |
| 571 |
return true; |
| 572 |
} |
| 573 |
|
| 574 |
template<int T> |
| 575 |
void LibcInfoWithPatchFunctions<T>::Unpatch() { |
| 576 |
// We have to cast our GenericFnPtrs to void* for unpatch. This is |
| 577 |
// contra the C++ spec; we use C-style casts to empahsize that. |
| 578 |
for (int i = 0; i < kNumFunctions; i++) { |
| 579 |
if (windows_fn_[i]) |
| 580 |
CHECK_EQ(sidestep::SIDESTEP_SUCCESS, |
| 581 |
PreamblePatcher::Unpatch((void*)windows_fn_[i], |
| 582 |
(void*)perftools_fn_[i], |
| 583 |
(void*)origstub_fn_[i])); |
| 584 |
} |
| 585 |
set_is_valid(false); |
| 586 |
} |
| 587 |
|
| 588 |
void WindowsInfo::Patch() { |
| 589 |
HMODULE hkernel32 = ::GetModuleHandleA("kernel32"); |
| 590 |
CHECK_NE(hkernel32, NULL); |
| 591 |
|
| 592 |
// Unlike for libc, we know these exist in our module, so we can get |
| 593 |
// and patch at the same time. |
| 594 |
for (int i = 0; i < kNumFunctions; i++) { |
| 595 |
function_info_[i].windows_fn = (GenericFnPtr) |
| 596 |
::GetProcAddress(hkernel32, function_info_[i].name); |
| 597 |
// If origstub_fn is not NULL, it's left around from a previous |
| 598 |
// patch. We need to set it to NULL for the new Patch call. |
| 599 |
// Since we've patched Unpatch() not to delete origstub_fn_ (it |
| 600 |
// causes problems in some contexts, though obviously not this |
| 601 |
// one), we should delete it now, before setting it to NULL. |
| 602 |
// NOTE: casting from a function to a pointer is contra the C++ |
| 603 |
// spec. It's not safe on IA64, but is on i386. We use |
| 604 |
// a C-style cast here to emphasize this is not legal C++. |
| 605 |
delete[] (char*)(function_info_[i].origstub_fn); |
| 606 |
function_info_[i].origstub_fn = NULL; // Patch() will fill this in |
| 607 |
CHECK_EQ(sidestep::SIDESTEP_SUCCESS, |
| 608 |
PreamblePatcher::Patch(function_info_[i].windows_fn, |
| 609 |
function_info_[i].perftools_fn, |
| 610 |
&function_info_[i].origstub_fn)); |
| 611 |
} |
| 612 |
} |
| 613 |
|
| 614 |
void WindowsInfo::Unpatch() { |
| 615 |
// We have to cast our GenericFnPtrs to void* for unpatch. This is |
| 616 |
// contra the C++ spec; we use C-style casts to empahsize that. |
| 617 |
for (int i = 0; i < kNumFunctions; i++) { |
| 618 |
CHECK_EQ(sidestep::SIDESTEP_SUCCESS, |
| 619 |
PreamblePatcher::Unpatch((void*)function_info_[i].windows_fn, |
| 620 |
(void*)function_info_[i].perftools_fn, |
| 621 |
(void*)function_info_[i].origstub_fn)); |
| 622 |
} |
| 623 |
} |
| 624 |
|
| 625 |
// You should hold the patch_all_modules_lock when calling this. |
| 626 |
void PatchOneModuleLocked(const LibcInfo& me_info) { |
| 627 |
// If we don't already have info on this module, let's add it. This |
| 628 |
// is where we're sad that each libcX has a different type, so we |
| 629 |
// can't use an array; instead, we have to use a switch statement. |
| 630 |
// Patch() returns false if there were no libc functions in the module. |
| 631 |
for (int i = 0; i < sizeof(g_module_libcs)/sizeof(*g_module_libcs); i++) { |
| 632 |
if (!g_module_libcs[i]->is_valid()) { // found an empty spot to add! |
| 633 |
switch (i) { |
| 634 |
case 0: libc1.Patch(me_info); return; |
| 635 |
case 1: libc2.Patch(me_info); return; |
| 636 |
case 2: libc3.Patch(me_info); return; |
| 637 |
case 3: libc4.Patch(me_info); return; |
| 638 |
case 4: libc5.Patch(me_info); return; |
| 639 |
case 5: libc6.Patch(me_info); return; |
| 640 |
case 6: libc7.Patch(me_info); return; |
| 641 |
case 7: libc8.Patch(me_info); return; |
| 642 |
} |
| 643 |
} |
| 644 |
} |
| 645 |
printf("PERFTOOLS ERROR: Too many modules containing libc in this executable\n"); |
| 646 |
} |
| 647 |
|
| 648 |
void PatchMainExecutableLocked() { |
| 649 |
if (main_executable.patched()) |
| 650 |
return; // main executable has already been patched |
| 651 |
ModuleEntryCopy fake_module_entry; // make a fake one to pass into Patch() |
| 652 |
// No need to call PopulateModuleEntryProcAddresses on the main executable. |
| 653 |
main_executable.PopulateWindowsFn(fake_module_entry); |
| 654 |
main_executable.Patch(main_executable); |
| 655 |
} |
| 656 |
|
| 657 |
// This lock is subject to a subtle and annoying lock inversion |
| 658 |
// problem: it may interact badly with unknown internal windows locks. |
| 659 |
// In particular, windows may be holding a lock when it calls |
| 660 |
// LoadLibraryExW and FreeLibrary, which we've patched. We have those |
| 661 |
// routines call PatchAllModules, which acquires this lock. If we |
| 662 |
// make windows system calls while holding this lock, those system |
| 663 |
// calls may need the internal windows locks that are being held in |
| 664 |
// the call to LoadLibraryExW, resulting in deadlock. The solution is |
| 665 |
// to be very careful not to call *any* windows routines while holding |
| 666 |
// patch_all_modules_lock, inside PatchAllModules(). |
| 667 |
static SpinLock patch_all_modules_lock(SpinLock::LINKER_INITIALIZED); |
| 668 |
|
| 669 |
// last_loaded: The set of modules that were loaded the last time |
| 670 |
// PatchAllModules was called. This is an optimization for only |
| 671 |
// looking at modules that were added or removed from the last call. |
| 672 |
static std::set<HMODULE> *g_last_loaded; |
| 673 |
|
| 674 |
// Iterates over all the modules currently loaded by the executable, |
| 675 |
// according to windows, and makes sure they're all patched. Most |
| 676 |
// modules will already be in loaded_modules, meaning we have already |
| 677 |
// loaded and either patched them or determined they did not need to |
| 678 |
// be patched. Others will not, which means we need to patch them |
| 679 |
// (if necessary). Finally, we have to go through the existing |
| 680 |
// g_module_libcs and see if any of those are *not* in the modules |
| 681 |
// currently loaded by the executable. If so, we need to invalidate |
| 682 |
// them. Returns true if we did any work (patching or invalidating), |
| 683 |
// false if we were a noop. May update loaded_modules as well. |
| 684 |
// NOTE: you must hold the patch_all_modules_lock to access loaded_modules. |
| 685 |
bool PatchAllModules() { |
| 686 |
std::vector<ModuleEntryCopy> modules; |
| 687 |
bool made_changes = false; |
| 688 |
|
| 689 |
const HANDLE hCurrentProcess = GetCurrentProcess(); |
| 690 |
DWORD num_modules = 0; |
| 691 |
HMODULE hModules[kMaxModules]; // max # of modules we support in one process |
| 692 |
if (!::EnumProcessModules(hCurrentProcess, hModules, sizeof(hModules), |
| 693 |
&num_modules)) { |
| 694 |
num_modules = 0; |
| 695 |
} |
| 696 |
// EnumProcessModules actually set the bytes written into hModules, |
| 697 |
// so we need to divide to make num_modules actually be a module-count. |
| 698 |
num_modules /= sizeof(*hModules); |
| 699 |
if (num_modules >= kMaxModules) { |
| 700 |
printf("PERFTOOLS ERROR: Too many modules in this executable to try" |
| 701 |
" to patch them all (if you need to, raise kMaxModules in" |
| 702 |
" patch_functions.cc).\n"); |
| 703 |
num_modules = kMaxModules; |
| 704 |
} |
| 705 |
|
| 706 |
// Now we handle the unpatching of modules we have in g_module_libcs |
| 707 |
// but that were not found in EnumProcessModules. We need to |
| 708 |
// invalidate them. To speed that up, we store the EnumProcessModules |
| 709 |
// output in a set. |
| 710 |
// At the same time, we prepare for the adding of new modules, by |
| 711 |
// removing from hModules all the modules we know we've already |
| 712 |
// patched (or decided don't need to be patched). At the end, |
| 713 |
// hModules will hold only the modules that we need to consider patching. |
| 714 |
std::set<HMODULE> currently_loaded_modules; |
| 715 |
{ |
| 716 |
SpinLockHolder h(&patch_all_modules_lock); |
| 717 |
if (!g_last_loaded) g_last_loaded = new std::set<HMODULE>; |
| 718 |
// At the end of this loop, currently_loaded_modules contains the |
| 719 |
// full list of EnumProcessModules, and hModules just the ones we |
| 720 |
// haven't handled yet. |
| 721 |
for (int i = 0; i < num_modules; ) { |
| 722 |
currently_loaded_modules.insert(hModules[i]); |
| 723 |
if (g_last_loaded->count(hModules[i]) > 0) { |
| 724 |
hModules[i] = hModules[--num_modules]; // replace element i with tail |
| 725 |
} else { |
| 726 |
i++; // keep element i |
| 727 |
} |
| 728 |
} |
| 729 |
// Now we do the unpatching/invalidation. |
| 730 |
for (int i = 0; i < sizeof(g_module_libcs)/sizeof(*g_module_libcs); i++) { |
| 731 |
if (g_module_libcs[i]->patched() && |
| 732 |
currently_loaded_modules.count(g_module_libcs[i]->hmodule()) == 0) { |
| 733 |
// Means g_module_libcs[i] is no longer loaded (no me32 matched). |
| 734 |
// We could call Unpatch() here, but why bother? The module |
| 735 |
// has gone away, so nobody is going to call into it anyway. |
| 736 |
g_module_libcs[i]->set_is_valid(false); |
| 737 |
made_changes = true; |
| 738 |
} |
| 739 |
} |
| 740 |
// Update the loaded module cache. |
| 741 |
g_last_loaded->swap(currently_loaded_modules); |
| 742 |
} |
| 743 |
|
| 744 |
// Now that we know what modules are new, let's get the info we'll |
| 745 |
// need to patch them. Note this *cannot* be done while holding the |
| 746 |
// lock, since it needs to make windows calls (see the lock-inversion |
| 747 |
// comments before the definition of patch_all_modules_lock). |
| 748 |
MODULEINFO mi; |
| 749 |
for (int i = 0; i < num_modules; i++) { |
| 750 |
if (::GetModuleInformation(hCurrentProcess, hModules[i], &mi, sizeof(mi))) |
| 751 |
modules.push_back(ModuleEntryCopy(mi)); |
| 752 |
} |
| 753 |
|
| 754 |
// Now we can do the patching of new modules. |
| 755 |
{ |
| 756 |
SpinLockHolder h(&patch_all_modules_lock); |
| 757 |
for (std::vector<ModuleEntryCopy>::iterator it = modules.begin(); |
| 758 |
it != modules.end(); ++it) { |
| 759 |
LibcInfo libc_info; |
| 760 |
if (libc_info.PopulateWindowsFn(*it)) { // true==module has libc routines |
| 761 |
PatchOneModuleLocked(libc_info); |
| 762 |
made_changes = true; |
| 763 |
} |
| 764 |
} |
| 765 |
|
| 766 |
// Now that we've dealt with the modules (dlls), update the main |
| 767 |
// executable. We do this last because PatchMainExecutableLocked |
| 768 |
// wants to look at how other modules were patched. |
| 769 |
if (!main_executable.patched()) { |
| 770 |
PatchMainExecutableLocked(); |
| 771 |
made_changes = true; |
| 772 |
} |
| 773 |
} |
| 774 |
// TODO(csilvers): for this to be reliable, we need to also take |
| 775 |
// into account if we *would* have patched any modules had they not |
| 776 |
// already been loaded. (That is, made_changes should ignore |
| 777 |
// g_last_loaded.) |
| 778 |
return made_changes; |
| 779 |
} |
| 780 |
|
| 781 |
|
| 782 |
} // end unnamed namespace |
| 783 |
|
| 784 |
// --------------------------------------------------------------------- |
| 785 |
// Now that we've done all the patching machinery, let's actually |
| 786 |
// define the functions we're patching in. Mostly these are |
| 787 |
// simple wrappers around the do_* routines in tcmalloc.cc. |
| 788 |
// |
| 789 |
// In fact, we #include tcmalloc.cc to get at the tcmalloc internal |
| 790 |
// do_* functions, the better to write our own hook functions. |
| 791 |
// U-G-L-Y, I know. But the alternatives are, perhaps, worse. This |
| 792 |
// also lets us define _msize(), _expand(), and other windows-specific |
| 793 |
// functions here, using tcmalloc internals, without polluting |
| 794 |
// tcmalloc.cc. |
| 795 |
// ------------------------------------------------------------------- |
| 796 |
|
| 797 |
// TODO(csilvers): refactor tcmalloc.cc into two files, so I can link |
| 798 |
// against the file with do_malloc, and ignore the one with malloc. |
| 799 |
#include "tcmalloc.cc" |
| 800 |
|
| 801 |
template<int T> |
| 802 |
void* LibcInfoWithPatchFunctions<T>::Perftools_malloc(size_t size) __THROW { |
| 803 |
return malloc_fast_path<tcmalloc::malloc_oom>(size); |
| 804 |
} |
| 805 |
|
| 806 |
template<int T> |
| 807 |
void LibcInfoWithPatchFunctions<T>::Perftools_free(void* ptr) __THROW { |
| 808 |
MallocHook::InvokeDeleteHook(ptr); |
| 809 |
// This calls the windows free if do_free decides ptr was not |
| 810 |
// allocated by tcmalloc. Note it calls the origstub_free from |
| 811 |
// *this* templatized instance of LibcInfo. See "template |
| 812 |
// trickiness" above. |
| 813 |
do_free_with_callback(ptr, (void (*)(void*))origstub_fn_[kFree], false, 0); |
| 814 |
} |
| 815 |
|
| 816 |
template<int T> |
| 817 |
void LibcInfoWithPatchFunctions<T>::Perftools_free_base(void* ptr) __THROW{ |
| 818 |
MallocHook::InvokeDeleteHook(ptr); |
| 819 |
// This calls the windows free if do_free decides ptr was not |
| 820 |
// allocated by tcmalloc. Note it calls the origstub_free from |
| 821 |
// *this* templatized instance of LibcInfo. See "template |
| 822 |
// trickiness" above. |
| 823 |
do_free_with_callback(ptr, (void(*)(void*))origstub_fn_[kFreeBase], false, 0); |
| 824 |
} |
| 825 |
|
| 826 |
template<int T> |
| 827 |
void LibcInfoWithPatchFunctions<T>::Perftools_free_dbg(void* ptr, int block_use) __THROW { |
| 828 |
MallocHook::InvokeDeleteHook(ptr); |
| 829 |
// The windows _free_dbg is called if ptr isn't owned by tcmalloc. |
| 830 |
if (MallocExtension::instance()->GetOwnership(ptr) == MallocExtension::kOwned) { |
| 831 |
do_free(ptr); |
| 832 |
} else { |
| 833 |
reinterpret_cast<void (*)(void*, int)>(origstub_fn_[kFreeDbg])(ptr, block_use); |
| 834 |
} |
| 835 |
} |
| 836 |
|
| 837 |
template<int T> |
| 838 |
void* LibcInfoWithPatchFunctions<T>::Perftools_realloc( |
| 839 |
void* old_ptr, size_t new_size) __THROW { |
| 840 |
if (old_ptr == NULL) { |
| 841 |
void* result = do_malloc_or_cpp_alloc(new_size); |
| 842 |
MallocHook::InvokeNewHook(result, new_size); |
| 843 |
return result; |
| 844 |
} |
| 845 |
if (new_size == 0) { |
| 846 |
MallocHook::InvokeDeleteHook(old_ptr); |
| 847 |
do_free_with_callback(old_ptr, |
| 848 |
(void (*)(void*))origstub_fn_[kFree], false, 0); |
| 849 |
return NULL; |
| 850 |
} |
| 851 |
return do_realloc_with_callback( |
| 852 |
old_ptr, new_size, |
| 853 |
(void (*)(void*))origstub_fn_[kFree], |
| 854 |
(size_t (*)(const void*))origstub_fn_[k_Msize]); |
| 855 |
} |
| 856 |
|
| 857 |
template<int T> |
| 858 |
void* LibcInfoWithPatchFunctions<T>::Perftools_calloc( |
| 859 |
size_t n, size_t elem_size) __THROW { |
| 860 |
void* result = do_calloc(n, elem_size); |
| 861 |
MallocHook::InvokeNewHook(result, n * elem_size); |
| 862 |
return result; |
| 863 |
} |
| 864 |
|
| 865 |
template<int T> |
| 866 |
void* LibcInfoWithPatchFunctions<T>::Perftools_new(size_t size) { |
| 867 |
return malloc_fast_path<tcmalloc::cpp_throw_oom>(size); |
| 868 |
} |
| 869 |
|
| 870 |
template<int T> |
| 871 |
void* LibcInfoWithPatchFunctions<T>::Perftools_newarray(size_t size) { |
| 872 |
return malloc_fast_path<tcmalloc::cpp_throw_oom>(size); |
| 873 |
} |
| 874 |
|
| 875 |
template<int T> |
| 876 |
void LibcInfoWithPatchFunctions<T>::Perftools_delete(void *p) { |
| 877 |
MallocHook::InvokeDeleteHook(p); |
| 878 |
do_free_with_callback(p, (void (*)(void*))origstub_fn_[kFree], false, 0); |
| 879 |
} |
| 880 |
|
| 881 |
template<int T> |
| 882 |
void LibcInfoWithPatchFunctions<T>::Perftools_deletearray(void *p) { |
| 883 |
MallocHook::InvokeDeleteHook(p); |
| 884 |
do_free_with_callback(p, (void (*)(void*))origstub_fn_[kFree], false, 0); |
| 885 |
} |
| 886 |
|
| 887 |
template<int T> |
| 888 |
void* LibcInfoWithPatchFunctions<T>::Perftools_new_nothrow( |
| 889 |
size_t size, const std::nothrow_t&) __THROW { |
| 890 |
return malloc_fast_path<tcmalloc::cpp_nothrow_oom>(size); |
| 891 |
} |
| 892 |
|
| 893 |
template<int T> |
| 894 |
void* LibcInfoWithPatchFunctions<T>::Perftools_newarray_nothrow( |
| 895 |
size_t size, const std::nothrow_t&) __THROW { |
| 896 |
return malloc_fast_path<tcmalloc::cpp_nothrow_oom>(size); |
| 897 |
} |
| 898 |
|
| 899 |
template<int T> |
| 900 |
void LibcInfoWithPatchFunctions<T>::Perftools_delete_nothrow( |
| 901 |
void *p, const std::nothrow_t&) __THROW { |
| 902 |
MallocHook::InvokeDeleteHook(p); |
| 903 |
do_free_with_callback(p, (void (*)(void*))origstub_fn_[kFree], false, 0); |
| 904 |
} |
| 905 |
|
| 906 |
template<int T> |
| 907 |
void LibcInfoWithPatchFunctions<T>::Perftools_deletearray_nothrow( |
| 908 |
void *p, const std::nothrow_t&) __THROW { |
| 909 |
MallocHook::InvokeDeleteHook(p); |
| 910 |
do_free_with_callback(p, (void (*)(void*))origstub_fn_[kFree], false, 0); |
| 911 |
} |
| 912 |
|
| 913 |
|
| 914 |
// _msize() lets you figure out how much space is reserved for a |
| 915 |
// pointer, in Windows. Even if applications don't call it, any DLL |
| 916 |
// with global constructors will call (transitively) something called |
| 917 |
// __dllonexit_lk in order to make sure the destructors get called |
| 918 |
// when the dll unloads. And that will call msize -- horrible things |
| 919 |
// can ensue if this is not hooked. Other parts of libc may also call |
| 920 |
// this internally. |
| 921 |
|
| 922 |
template<int T> |
| 923 |
size_t LibcInfoWithPatchFunctions<T>::Perftools__msize(void* ptr) __THROW { |
| 924 |
return GetSizeWithCallback(ptr, (size_t (*)(const void*))origstub_fn_[k_Msize]); |
| 925 |
} |
| 926 |
|
| 927 |
LPVOID WINAPI WindowsInfo::Perftools_HeapAlloc(HANDLE hHeap, DWORD dwFlags, |
| 928 |
DWORD_PTR dwBytes) { |
| 929 |
LPVOID result = ((LPVOID (WINAPI *)(HANDLE, DWORD, DWORD_PTR)) |
| 930 |
function_info_[kHeapAlloc].origstub_fn)( |
| 931 |
hHeap, dwFlags, dwBytes); |
| 932 |
MallocHook::InvokeNewHook(result, dwBytes); |
| 933 |
return result; |
| 934 |
} |
| 935 |
|
| 936 |
BOOL WINAPI WindowsInfo::Perftools_HeapFree(HANDLE hHeap, DWORD dwFlags, |
| 937 |
LPVOID lpMem) { |
| 938 |
MallocHook::InvokeDeleteHook(lpMem); |
| 939 |
return ((BOOL (WINAPI *)(HANDLE, DWORD, LPVOID)) |
| 940 |
function_info_[kHeapFree].origstub_fn)( |
| 941 |
hHeap, dwFlags, lpMem); |
| 942 |
} |
| 943 |
|
| 944 |
HMODULE WINAPI WindowsInfo::Perftools_LoadLibraryExW(LPCWSTR lpFileName, |
| 945 |
HANDLE hFile, |
| 946 |
DWORD dwFlags) { |
| 947 |
HMODULE rv; |
| 948 |
// Check to see if the modules is already loaded, flag 0 gets a |
| 949 |
// reference if it was loaded. If it was loaded no need to call |
| 950 |
// PatchAllModules, just increase the reference count to match |
| 951 |
// what GetModuleHandleExW does internally inside windows. |
| 952 |
if (::GetModuleHandleExW(0, lpFileName, &rv)) { |
| 953 |
return rv; |
| 954 |
} else { |
| 955 |
// Not already loaded, so load it. |
| 956 |
rv = ((HMODULE (WINAPI *)(LPCWSTR, HANDLE, DWORD)) |
| 957 |
function_info_[kLoadLibraryExW].origstub_fn)( |
| 958 |
lpFileName, hFile, dwFlags); |
| 959 |
// This will patch any newly loaded libraries, if patching needs |
| 960 |
// to be done. |
| 961 |
PatchAllModules(); |
| 962 |
|
| 963 |
return rv; |
| 964 |
} |
| 965 |
} |
| 966 |
|
| 967 |
BOOL WINAPI WindowsInfo::Perftools_FreeLibrary(HMODULE hLibModule) { |
| 968 |
BOOL rv = ((BOOL (WINAPI *)(HMODULE)) |
| 969 |
function_info_[kFreeLibrary].origstub_fn)(hLibModule); |
| 970 |
|
| 971 |
// Check to see if the module is still loaded by passing the base |
| 972 |
// address and seeing if it comes back with the same address. If it |
| 973 |
// is the same address it's still loaded, so the FreeLibrary() call |
| 974 |
// was a noop, and there's no need to redo the patching. |
| 975 |
HMODULE owner = NULL; |
| 976 |
BOOL result = ::GetModuleHandleExW( |
| 977 |
(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | |
| 978 |
GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT), |
| 979 |
(LPCWSTR)hLibModule, |
| 980 |
&owner); |
| 981 |
if (result && owner == hLibModule) |
| 982 |
return rv; |
| 983 |
|
| 984 |
PatchAllModules(); // this will fix up the list of patched libraries |
| 985 |
return rv; |
| 986 |
} |
| 987 |
|
| 988 |
|
| 989 |
// --------------------------------------------------------------------- |
| 990 |
// PatchWindowsFunctions() |
| 991 |
// This is the function that is exposed to the outside world. |
| 992 |
// It should be called before the program becomes multi-threaded, |
| 993 |
// since main_executable_windows.Patch() is not thread-safe. |
| 994 |
// --------------------------------------------------------------------- |
| 995 |
|
| 996 |
void PatchWindowsFunctions() { |
| 997 |
const char* disable_env = TCMallocGetenvSafe("TCMALLOC_DISABLE_REPLACEMENT"); |
| 998 |
const bool should_skip = disable_env && |
| 999 |
disable_env[0] == '1' && |
| 1000 |
disable_env[1] == '\0'; |
| 1001 |
|
| 1002 |
if (!should_skip) { |
| 1003 |
// This does the libc patching in every module, and the main executable. |
| 1004 |
PatchAllModules(); |
| 1005 |
main_executable_windows.Patch(); |
| 1006 |
} |
| 1007 |
} |
| 1008 |
|
| 1009 |
#if 0 |
| 1010 |
// It's possible to unpatch all the functions when we are exiting. |
| 1011 |
|
| 1012 |
// The idea is to handle properly windows-internal data that is |
| 1013 |
// allocated before PatchWindowsFunctions is called. If all |
| 1014 |
// destruction happened in reverse order from construction, then we |
| 1015 |
// could call UnpatchWindowsFunctions at just the right time, so that |
| 1016 |
// that early-allocated data would be freed using the windows |
| 1017 |
// allocation functions rather than tcmalloc. The problem is that |
| 1018 |
// windows allocates some structures lazily, so it would allocate them |
| 1019 |
// late (using tcmalloc) and then try to deallocate them late as well. |
| 1020 |
// So instead of unpatching, we just modify all the tcmalloc routines |
| 1021 |
// so they call through to the libc rountines if the memory in |
| 1022 |
// question doesn't seem to have been allocated with tcmalloc. I keep |
| 1023 |
// this unpatch code around for reference. |
| 1024 |
|
| 1025 |
void UnpatchWindowsFunctions() { |
| 1026 |
// We need to go back to the system malloc/etc at global destruct time, |
| 1027 |
// so objects that were constructed before tcmalloc, using the system |
| 1028 |
// malloc, can destroy themselves using the system free. This depends |
| 1029 |
// on DLLs unloading in the reverse order in which they load! |
| 1030 |
// |
| 1031 |
// We also go back to the default HeapAlloc/etc, just for consistency. |
| 1032 |
// Who knows, it may help avoid weird bugs in some situations. |
| 1033 |
main_executable_windows.Unpatch(); |
| 1034 |
main_executable.Unpatch(); |
| 1035 |
if (libc1.is_valid()) libc1.Unpatch(); |
| 1036 |
if (libc2.is_valid()) libc2.Unpatch(); |
| 1037 |
if (libc3.is_valid()) libc3.Unpatch(); |
| 1038 |
if (libc4.is_valid()) libc4.Unpatch(); |
| 1039 |
if (libc5.is_valid()) libc5.Unpatch(); |
| 1040 |
if (libc6.is_valid()) libc6.Unpatch(); |
| 1041 |
if (libc7.is_valid()) libc7.Unpatch(); |
| 1042 |
if (libc8.is_valid()) libc8.Unpatch(); |
| 1043 |
} |
| 1044 |
#endif |