| 1 |
//===-- sanitizer_linux_libcdep.cpp ---------------------------------------===// |
| 2 |
// |
| 3 |
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 |
// See https://llvm.org/LICENSE.txt for license information. |
| 5 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 |
// |
| 7 |
//===----------------------------------------------------------------------===// |
| 8 |
// |
| 9 |
// This file is shared between AddressSanitizer and ThreadSanitizer |
| 10 |
// run-time libraries and implements linux-specific functions from |
| 11 |
// sanitizer_libc.h. |
| 12 |
//===----------------------------------------------------------------------===// |
| 13 |
|
| 14 |
#include "sanitizer_platform.h" |
| 15 |
|
| 16 |
#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \ |
| 17 |
SANITIZER_SOLARIS |
| 18 |
|
| 19 |
# include "sanitizer_allocator_internal.h" |
| 20 |
# include "sanitizer_atomic.h" |
| 21 |
# include "sanitizer_common.h" |
| 22 |
# include "sanitizer_file.h" |
| 23 |
# include "sanitizer_flags.h" |
| 24 |
# include "sanitizer_getauxval.h" |
| 25 |
# include "sanitizer_glibc_version.h" |
| 26 |
# include "sanitizer_linux.h" |
| 27 |
# include "sanitizer_placement_new.h" |
| 28 |
# include "sanitizer_procmaps.h" |
| 29 |
# include "sanitizer_solaris.h" |
| 30 |
|
| 31 |
# if SANITIZER_NETBSD |
| 32 |
# define _RTLD_SOURCE // for __lwp_gettcb_fast() / __lwp_getprivate_fast() |
| 33 |
# endif |
| 34 |
|
| 35 |
# include <dlfcn.h> // for dlsym() |
| 36 |
# include <link.h> |
| 37 |
# include <pthread.h> |
| 38 |
# include <signal.h> |
| 39 |
# include <sys/mman.h> |
| 40 |
# include <sys/resource.h> |
| 41 |
# include <syslog.h> |
| 42 |
|
| 43 |
# if SANITIZER_GLIBC |
| 44 |
# include <gnu/libc-version.h> |
| 45 |
# endif |
| 46 |
|
| 47 |
# if !defined(ElfW) |
| 48 |
# define ElfW(type) Elf_##type |
| 49 |
# endif |
| 50 |
|
| 51 |
# if SANITIZER_FREEBSD |
| 52 |
# include <pthread_np.h> |
| 53 |
# include <sys/auxv.h> |
| 54 |
# include <sys/sysctl.h> |
| 55 |
# define pthread_getattr_np pthread_attr_get_np |
| 56 |
// The MAP_NORESERVE define has been removed in FreeBSD 11.x, and even before |
| 57 |
// that, it was never implemented. So just define it to zero. |
| 58 |
# undef MAP_NORESERVE |
| 59 |
# define MAP_NORESERVE 0 |
| 60 |
extern const Elf_Auxinfo *__elf_aux_vector; |
| 61 |
extern "C" int __sys_sigaction(int signum, const struct sigaction *act, |
| 62 |
struct sigaction *oldact); |
| 63 |
# endif |
| 64 |
|
| 65 |
# if SANITIZER_NETBSD |
| 66 |
# include <lwp.h> |
| 67 |
# include <sys/sysctl.h> |
| 68 |
# include <sys/tls.h> |
| 69 |
# endif |
| 70 |
|
| 71 |
# if SANITIZER_SOLARIS |
| 72 |
# include <stddef.h> |
| 73 |
# include <stdlib.h> |
| 74 |
# include <thread.h> |
| 75 |
# endif |
| 76 |
|
| 77 |
# if SANITIZER_ANDROID |
| 78 |
# include <android/api-level.h> |
| 79 |
# if !defined(CPU_COUNT) && !defined(__aarch64__) |
| 80 |
# include <dirent.h> |
| 81 |
# include <fcntl.h> |
| 82 |
struct __sanitizer::linux_dirent { |
| 83 |
long d_ino; |
| 84 |
off_t d_off; |
| 85 |
unsigned short d_reclen; |
| 86 |
char d_name[]; |
| 87 |
}; |
| 88 |
# endif |
| 89 |
# endif |
| 90 |
|
| 91 |
# if !SANITIZER_ANDROID |
| 92 |
# include <elf.h> |
| 93 |
# include <unistd.h> |
| 94 |
# endif |
| 95 |
|
| 96 |
namespace __sanitizer { |
| 97 |
|
| 98 |
SANITIZER_WEAK_ATTRIBUTE int real_sigaction(int signum, const void *act, |
| 99 |
void *oldact); |
| 100 |
|
| 101 |
int internal_sigaction(int signum, const void *act, void *oldact) { |
| 102 |
# if SANITIZER_FREEBSD |
| 103 |
// On FreeBSD, call the sigaction syscall directly (part of libsys in FreeBSD |
| 104 |
// 15) since the libc version goes via a global interposing table. Due to |
| 105 |
// library initialization order the table can be relocated after the call to |
| 106 |
// InitializeDeadlySignals() which then crashes when dereferencing the |
| 107 |
// uninitialized pointer in libc. |
| 108 |
return __sys_sigaction(signum, (const struct sigaction *)act, |
| 109 |
(struct sigaction *)oldact); |
| 110 |
# else |
| 111 |
# if !SANITIZER_GO |
| 112 |
if (&real_sigaction) |
| 113 |
return real_sigaction(signum, act, oldact); |
| 114 |
# endif |
| 115 |
return sigaction(signum, (const struct sigaction *)act, |
| 116 |
(struct sigaction *)oldact); |
| 117 |
# endif |
| 118 |
} |
| 119 |
|
| 120 |
void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top, |
| 121 |
uptr *stack_bottom) { |
| 122 |
CHECK(stack_top); |
| 123 |
CHECK(stack_bottom); |
| 124 |
if (at_initialization) { |
| 125 |
// This is the main thread. Libpthread may not be initialized yet. |
| 126 |
struct rlimit rl; |
| 127 |
CHECK_EQ(getrlimit(RLIMIT_STACK, &rl), 0); |
| 128 |
|
| 129 |
// Find the mapping that contains a stack variable. |
| 130 |
MemoryMappingLayout proc_maps(/*cache_enabled*/ true); |
| 131 |
if (proc_maps.Error()) { |
| 132 |
*stack_top = *stack_bottom = 0; |
| 133 |
return; |
| 134 |
} |
| 135 |
MemoryMappedSegment segment; |
| 136 |
uptr prev_end = 0; |
| 137 |
while (proc_maps.Next(&segment)) { |
| 138 |
if ((uptr)&rl < segment.end) |
| 139 |
break; |
| 140 |
prev_end = segment.end; |
| 141 |
} |
| 142 |
CHECK((uptr)&rl >= segment.start && (uptr)&rl < segment.end); |
| 143 |
|
| 144 |
// Get stacksize from rlimit, but clip it so that it does not overlap |
| 145 |
// with other mappings. |
| 146 |
uptr stacksize = rl.rlim_cur; |
| 147 |
if (stacksize > segment.end - prev_end) |
| 148 |
stacksize = segment.end - prev_end; |
| 149 |
// When running with unlimited stack size, we still want to set some limit. |
| 150 |
// The unlimited stack size is caused by 'ulimit -s unlimited'. |
| 151 |
// Also, for some reason, GNU make spawns subprocesses with unlimited stack. |
| 152 |
if (stacksize > kMaxThreadStackSize) |
| 153 |
stacksize = kMaxThreadStackSize; |
| 154 |
*stack_top = segment.end; |
| 155 |
*stack_bottom = segment.end - stacksize; |
| 156 |
|
| 157 |
uptr maxAddr = GetMaxUserVirtualAddress(); |
| 158 |
// Edge case: the stack mapping on some systems may be off-by-one e.g., |
| 159 |
// fffffffdf000-1000000000000 rw-p 00000000 00:00 0 [stack] |
| 160 |
// instead of: |
| 161 |
// fffffffdf000- ffffffffffff |
| 162 |
// The out-of-range stack_top can result in an invalid shadow address |
| 163 |
// calculation, since those usually assume the parameters are in range. |
| 164 |
if (*stack_top == maxAddr + 1) |
| 165 |
*stack_top = maxAddr; |
| 166 |
else |
| 167 |
CHECK_LE(*stack_top, maxAddr); |
| 168 |
|
| 169 |
return; |
| 170 |
} |
| 171 |
uptr stacksize = 0; |
| 172 |
void *stackaddr = nullptr; |
| 173 |
# if SANITIZER_SOLARIS |
| 174 |
stack_t ss; |
| 175 |
CHECK_EQ(thr_stksegment(&ss), 0); |
| 176 |
stacksize = ss.ss_size; |
| 177 |
stackaddr = (char *)ss.ss_sp - stacksize; |
| 178 |
# else // !SANITIZER_SOLARIS |
| 179 |
pthread_attr_t attr; |
| 180 |
pthread_attr_init(&attr); |
| 181 |
CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0); |
| 182 |
internal_pthread_attr_getstack(&attr, &stackaddr, &stacksize); |
| 183 |
pthread_attr_destroy(&attr); |
| 184 |
# endif // SANITIZER_SOLARIS |
| 185 |
|
| 186 |
*stack_top = (uptr)stackaddr + stacksize; |
| 187 |
*stack_bottom = (uptr)stackaddr; |
| 188 |
} |
| 189 |
|
| 190 |
# if !SANITIZER_GO |
| 191 |
bool SetEnv(const char *name, const char *value) { |
| 192 |
void *f = dlsym(RTLD_NEXT, "setenv"); |
| 193 |
if (!f) |
| 194 |
return false; |
| 195 |
typedef int (*setenv_ft)(const char *name, const char *value, int overwrite); |
| 196 |
setenv_ft setenv_f; |
| 197 |
CHECK_EQ(sizeof(setenv_f), sizeof(f)); |
| 198 |
internal_memcpy(&setenv_f, &f, sizeof(f)); |
| 199 |
return setenv_f(name, value, 1) == 0; |
| 200 |
} |
| 201 |
# endif |
| 202 |
|
| 203 |
// True if we can use dlpi_tls_data. glibc before 2.25 may leave NULL (BZ |
| 204 |
// #19826) so dlpi_tls_data cannot be used. |
| 205 |
// |
| 206 |
// musl before 1.2.3 and FreeBSD as of 12.2 incorrectly set dlpi_tls_data to |
| 207 |
// the TLS initialization image |
| 208 |
// https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=254774 |
| 209 |
__attribute__((unused)) static int g_use_dlpi_tls_data; |
| 210 |
|
| 211 |
# if SANITIZER_GLIBC && !SANITIZER_GO |
| 212 |
static void GetGLibcVersion(int *major, int *minor, int *patch) { |
| 213 |
const char *p = gnu_get_libc_version(); |
| 214 |
*major = internal_simple_strtoll(p, &p, 10); |
| 215 |
// Caller does not expect anything else. |
| 216 |
CHECK_EQ(*major, 2); |
| 217 |
*minor = (*p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0; |
| 218 |
*patch = (*p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0; |
| 219 |
} |
| 220 |
|
| 221 |
static uptr ThreadDescriptorSizeFallback() { |
| 222 |
# if defined(__x86_64__) || defined(__i386__) || defined(__arm__) || \ |
| 223 |
SANITIZER_RISCV64 |
| 224 |
int major; |
| 225 |
int minor; |
| 226 |
int patch; |
| 227 |
GetGLibcVersion(&major, &minor, &patch); |
| 228 |
# endif |
| 229 |
|
| 230 |
# if defined(__x86_64__) || defined(__i386__) || defined(__arm__) |
| 231 |
/* sizeof(struct pthread) values from various glibc versions. */ |
| 232 |
if (SANITIZER_X32) |
| 233 |
return 1728; // Assume only one particular version for x32. |
| 234 |
// For ARM sizeof(struct pthread) changed in Glibc 2.23. |
| 235 |
if (SANITIZER_ARM) |
| 236 |
return minor <= 22 ? 1120 : 1216; |
| 237 |
if (minor <= 3) |
| 238 |
return FIRST_32_SECOND_64(1104, 1696); |
| 239 |
if (minor == 4) |
| 240 |
return FIRST_32_SECOND_64(1120, 1728); |
| 241 |
if (minor == 5) |
| 242 |
return FIRST_32_SECOND_64(1136, 1728); |
| 243 |
if (minor <= 9) |
| 244 |
return FIRST_32_SECOND_64(1136, 1712); |
| 245 |
if (minor == 10) |
| 246 |
return FIRST_32_SECOND_64(1168, 1776); |
| 247 |
if (minor == 11 || (minor == 12 && patch == 1)) |
| 248 |
return FIRST_32_SECOND_64(1168, 2288); |
| 249 |
if (minor <= 14) |
| 250 |
return FIRST_32_SECOND_64(1168, 2304); |
| 251 |
if (minor < 32) // Unknown version |
| 252 |
return FIRST_32_SECOND_64(1216, 2304); |
| 253 |
// minor == 32 |
| 254 |
return FIRST_32_SECOND_64(1344, 2496); |
| 255 |
# endif |
| 256 |
|
| 257 |
# if SANITIZER_RISCV64 |
| 258 |
// TODO: consider adding an optional runtime check for an unknown (untested) |
| 259 |
// glibc version |
| 260 |
if (minor <= 28) // WARNING: the highest tested version is 2.29 |
| 261 |
return 1772; // no guarantees for this one |
| 262 |
if (minor <= 31) |
| 263 |
return 1772; // tested against glibc 2.29, 2.31 |
| 264 |
return 1936; // tested against glibc 2.32 |
| 265 |
# endif |
| 266 |
|
| 267 |
# if defined(__s390__) || defined(__sparc__) |
| 268 |
// The size of a prefix of TCB including pthread::{specific_1stblock,specific} |
| 269 |
// suffices. Just return offsetof(struct pthread, specific_used), which hasn't |
| 270 |
// changed since 2007-05. Technically this applies to i386/x86_64 as well but |
| 271 |
// we call _dl_get_tls_static_info and need the precise size of struct |
| 272 |
// pthread. |
| 273 |
return FIRST_32_SECOND_64(524, 1552); |
| 274 |
# endif |
| 275 |
|
| 276 |
# if defined(__mips__) |
| 277 |
// TODO(sagarthakur): add more values as per different glibc versions. |
| 278 |
return FIRST_32_SECOND_64(1152, 1776); |
| 279 |
# endif |
| 280 |
|
| 281 |
# if SANITIZER_LOONGARCH64 |
| 282 |
return 1856; // from glibc 2.36 |
| 283 |
# endif |
| 284 |
|
| 285 |
# if defined(__aarch64__) |
| 286 |
// The sizeof (struct pthread) is the same from GLIBC 2.17 to 2.22. |
| 287 |
return 1776; |
| 288 |
# endif |
| 289 |
|
| 290 |
# if defined(__powerpc64__) |
| 291 |
return 1776; // from glibc.ppc64le 2.20-8.fc21 |
| 292 |
# endif |
| 293 |
} |
| 294 |
# endif // SANITIZER_GLIBC && !SANITIZER_GO |
| 295 |
|
| 296 |
# if SANITIZER_FREEBSD && !SANITIZER_GO |
| 297 |
// FIXME: Implementation is very GLIBC specific, but it's used by FreeBSD. |
| 298 |
static uptr ThreadDescriptorSizeFallback() { |
| 299 |
# if defined(__s390__) || defined(__sparc__) |
| 300 |
// The size of a prefix of TCB including pthread::{specific_1stblock,specific} |
| 301 |
// suffices. Just return offsetof(struct pthread, specific_used), which hasn't |
| 302 |
// changed since 2007-05. Technically this applies to i386/x86_64 as well but |
| 303 |
// we call _dl_get_tls_static_info and need the precise size of struct |
| 304 |
// pthread. |
| 305 |
return FIRST_32_SECOND_64(524, 1552); |
| 306 |
# endif |
| 307 |
|
| 308 |
# if defined(__mips__) |
| 309 |
// TODO(sagarthakur): add more values as per different glibc versions. |
| 310 |
return FIRST_32_SECOND_64(1152, 1776); |
| 311 |
# endif |
| 312 |
|
| 313 |
# if SANITIZER_LOONGARCH64 |
| 314 |
return 1856; // from glibc 2.36 |
| 315 |
# endif |
| 316 |
|
| 317 |
# if defined(__aarch64__) |
| 318 |
// The sizeof (struct pthread) is the same from GLIBC 2.17 to 2.22. |
| 319 |
return 1776; |
| 320 |
# endif |
| 321 |
|
| 322 |
# if defined(__powerpc64__) |
| 323 |
return 1776; // from glibc.ppc64le 2.20-8.fc21 |
| 324 |
# endif |
| 325 |
|
| 326 |
return 0; |
| 327 |
} |
| 328 |
# endif // SANITIZER_FREEBSD && !SANITIZER_GO |
| 329 |
|
| 330 |
# if (SANITIZER_FREEBSD || SANITIZER_GLIBC) && !SANITIZER_GO |
| 331 |
// On glibc x86_64, ThreadDescriptorSize() needs to be precise due to the usage |
| 332 |
// of g_tls_size. On other targets, ThreadDescriptorSize() is only used by lsan |
| 333 |
// to get the pointer to thread-specific data keys in the thread control block. |
| 334 |
// sizeof(struct pthread) from glibc. |
| 335 |
static uptr thread_descriptor_size; |
| 336 |
|
| 337 |
uptr ThreadDescriptorSize() { return thread_descriptor_size; } |
| 338 |
|
| 339 |
# if SANITIZER_GLIBC |
| 340 |
__attribute__((unused)) static size_t g_tls_size; |
| 341 |
# endif |
| 342 |
|
| 343 |
void InitTlsSize() { |
| 344 |
# if SANITIZER_GLIBC |
| 345 |
int major, minor, patch; |
| 346 |
GetGLibcVersion(&major, &minor, &patch); |
| 347 |
g_use_dlpi_tls_data = major == 2 && minor >= 25; |
| 348 |
|
| 349 |
if (major == 2 && minor >= 34) { |
| 350 |
// _thread_db_sizeof_pthread is a GLIBC_PRIVATE symbol that is exported in |
| 351 |
// glibc 2.34 and later. |
| 352 |
if (unsigned *psizeof = static_cast<unsigned *>( |
| 353 |
dlsym(RTLD_DEFAULT, "_thread_db_sizeof_pthread"))) { |
| 354 |
thread_descriptor_size = *psizeof; |
| 355 |
} |
| 356 |
} |
| 357 |
|
| 358 |
# if defined(__aarch64__) || defined(__x86_64__) || \ |
| 359 |
defined(__powerpc64__) || defined(__loongarch__) |
| 360 |
auto *get_tls_static_info = (void (*)(size_t *, size_t *))dlsym( |
| 361 |
RTLD_DEFAULT, "_dl_get_tls_static_info"); |
| 362 |
size_t tls_align; |
| 363 |
// Can be null if static link. |
| 364 |
if (get_tls_static_info) |
| 365 |
get_tls_static_info(&g_tls_size, &tls_align); |
| 366 |
# endif |
| 367 |
|
| 368 |
# endif // SANITIZER_GLIBC |
| 369 |
|
| 370 |
if (!thread_descriptor_size) |
| 371 |
thread_descriptor_size = ThreadDescriptorSizeFallback(); |
| 372 |
} |
| 373 |
|
| 374 |
# if defined(__mips__) || defined(__powerpc64__) || SANITIZER_RISCV64 || \ |
| 375 |
SANITIZER_LOONGARCH64 |
| 376 |
// TlsPreTcbSize includes size of struct pthread_descr and size of tcb |
| 377 |
// head structure. It lies before the static tls blocks. |
| 378 |
static uptr TlsPreTcbSize() { |
| 379 |
# if defined(__mips__) |
| 380 |
const uptr kTcbHead = 16; // sizeof (tcbhead_t) |
| 381 |
# elif defined(__powerpc64__) |
| 382 |
const uptr kTcbHead = 88; // sizeof (tcbhead_t) |
| 383 |
# elif SANITIZER_RISCV64 |
| 384 |
const uptr kTcbHead = 16; // sizeof (tcbhead_t) |
| 385 |
# elif SANITIZER_LOONGARCH64 |
| 386 |
const uptr kTcbHead = 16; // sizeof (tcbhead_t) |
| 387 |
# endif |
| 388 |
const uptr kTlsAlign = 16; |
| 389 |
const uptr kTlsPreTcbSize = |
| 390 |
RoundUpTo(ThreadDescriptorSize() + kTcbHead, kTlsAlign); |
| 391 |
return kTlsPreTcbSize; |
| 392 |
} |
| 393 |
# endif |
| 394 |
# else // (SANITIZER_FREEBSD || SANITIZER_GLIBC) && !SANITIZER_GO |
| 395 |
void InitTlsSize() {} |
| 396 |
uptr ThreadDescriptorSize() { return 0; } |
| 397 |
# endif // (SANITIZER_FREEBSD || SANITIZER_GLIBC) && !SANITIZER_GO |
| 398 |
|
| 399 |
# if (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_SOLARIS) && \ |
| 400 |
!SANITIZER_ANDROID && !SANITIZER_GO |
| 401 |
namespace { |
| 402 |
struct TlsBlock { |
| 403 |
uptr begin, end, align; |
| 404 |
size_t tls_modid; |
| 405 |
bool operator<(const TlsBlock &rhs) const { return begin < rhs.begin; } |
| 406 |
}; |
| 407 |
} // namespace |
| 408 |
|
| 409 |
# ifdef __s390__ |
| 410 |
extern "C" uptr __tls_get_offset(void *arg); |
| 411 |
|
| 412 |
static uptr TlsGetOffset(uptr ti_module, uptr ti_offset) { |
| 413 |
// The __tls_get_offset ABI requires %r12 to point to GOT and %r2 to be an |
| 414 |
// offset of a struct tls_index inside GOT. We don't possess either of the |
| 415 |
// two, so violate the letter of the "ELF Handling For Thread-Local |
| 416 |
// Storage" document and assume that the implementation just dereferences |
| 417 |
// %r2 + %r12. |
| 418 |
uptr tls_index[2] = {ti_module, ti_offset}; |
| 419 |
register uptr r2 asm("2") = 0; |
| 420 |
register void *r12 asm("12") = tls_index; |
| 421 |
asm("basr %%r14, %[__tls_get_offset]" |
| 422 |
: "+r"(r2) |
| 423 |
: [__tls_get_offset] "r"(__tls_get_offset), "r"(r12) |
| 424 |
: "memory", "cc", "0", "1", "3", "4", "5", "14"); |
| 425 |
return r2; |
| 426 |
} |
| 427 |
# else |
| 428 |
extern "C" void *__tls_get_addr(size_t *); |
| 429 |
# endif |
| 430 |
|
| 431 |
static size_t main_tls_modid; |
| 432 |
|
| 433 |
static int CollectStaticTlsBlocks(struct dl_phdr_info *info, size_t size, |
| 434 |
void *data) { |
| 435 |
size_t tls_modid; |
| 436 |
# if SANITIZER_SOLARIS |
| 437 |
// dlpi_tls_modid is only available since Solaris 11.4 SRU 10. Use |
| 438 |
// dlinfo(RTLD_DI_LINKMAP) instead which works on all of Solaris 11.3, |
| 439 |
// 11.4, and Illumos. The tlsmodid of the executable was changed to 1 in |
| 440 |
// 11.4 to match other implementations. |
| 441 |
if (size >= offsetof(dl_phdr_info_test, dlpi_tls_modid)) |
| 442 |
main_tls_modid = 1; |
| 443 |
else |
| 444 |
main_tls_modid = 0; |
| 445 |
g_use_dlpi_tls_data = 0; |
| 446 |
Rt_map *map; |
| 447 |
dlinfo(RTLD_SELF, RTLD_DI_LINKMAP, &map); |
| 448 |
tls_modid = map->rt_tlsmodid; |
| 449 |
# else |
| 450 |
main_tls_modid = 1; |
| 451 |
tls_modid = info->dlpi_tls_modid; |
| 452 |
# endif |
| 453 |
|
| 454 |
if (tls_modid < main_tls_modid) |
| 455 |
return 0; |
| 456 |
uptr begin; |
| 457 |
# if !SANITIZER_SOLARIS |
| 458 |
begin = (uptr)info->dlpi_tls_data; |
| 459 |
# endif |
| 460 |
if (!g_use_dlpi_tls_data) { |
| 461 |
// Call __tls_get_addr as a fallback. This forces TLS allocation on glibc |
| 462 |
// and FreeBSD. |
| 463 |
# ifdef __s390__ |
| 464 |
begin = (uptr)__builtin_thread_pointer() + TlsGetOffset(tls_modid, 0); |
| 465 |
# else |
| 466 |
size_t mod_and_off[2] = {tls_modid, 0}; |
| 467 |
begin = (uptr)__tls_get_addr(mod_and_off); |
| 468 |
# endif |
| 469 |
} |
| 470 |
for (unsigned i = 0; i != info->dlpi_phnum; ++i) |
| 471 |
if (info->dlpi_phdr[i].p_type == PT_TLS) { |
| 472 |
static_cast<InternalMmapVector<TlsBlock> *>(data)->push_back( |
| 473 |
TlsBlock{begin, begin + info->dlpi_phdr[i].p_memsz, |
| 474 |
info->dlpi_phdr[i].p_align, tls_modid}); |
| 475 |
break; |
| 476 |
} |
| 477 |
return 0; |
| 478 |
} |
| 479 |
|
| 480 |
__attribute__((unused)) static void GetStaticTlsBoundary(uptr *addr, uptr *size, |
| 481 |
uptr *align) { |
| 482 |
InternalMmapVector<TlsBlock> ranges; |
| 483 |
dl_iterate_phdr(CollectStaticTlsBlocks, &ranges); |
| 484 |
uptr len = ranges.size(); |
| 485 |
Sort(ranges.begin(), len); |
| 486 |
// Find the range with tls_modid == main_tls_modid. For glibc, because |
| 487 |
// libc.so uses PT_TLS, this module is guaranteed to exist and is one of |
| 488 |
// the initially loaded modules. |
| 489 |
uptr one = 0; |
| 490 |
while (one != len && ranges[one].tls_modid != main_tls_modid) ++one; |
| 491 |
if (one == len) { |
| 492 |
// This may happen with musl if no module uses PT_TLS. |
| 493 |
*addr = 0; |
| 494 |
*size = 0; |
| 495 |
*align = 1; |
| 496 |
return; |
| 497 |
} |
| 498 |
// Find the maximum consecutive ranges. We consider two modules consecutive if |
| 499 |
// the gap is smaller than the alignment of the latter range. The dynamic |
| 500 |
// loader places static TLS blocks this way not to waste space. |
| 501 |
uptr l = one; |
| 502 |
*align = ranges[l].align; |
| 503 |
while (l != 0 && ranges[l].begin < ranges[l - 1].end + ranges[l].align) |
| 504 |
*align = Max(*align, ranges[--l].align); |
| 505 |
uptr r = one + 1; |
| 506 |
while (r != len && ranges[r].begin < ranges[r - 1].end + ranges[r].align) |
| 507 |
*align = Max(*align, ranges[r++].align); |
| 508 |
*addr = ranges[l].begin; |
| 509 |
*size = ranges[r - 1].end - ranges[l].begin; |
| 510 |
} |
| 511 |
# endif // (x86_64 || i386 || mips || ...) && (SANITIZER_FREEBSD || |
| 512 |
// SANITIZER_LINUX) && !SANITIZER_ANDROID && !SANITIZER_GO |
| 513 |
|
| 514 |
# if SANITIZER_NETBSD |
| 515 |
static struct tls_tcb *ThreadSelfTlsTcb() { |
| 516 |
struct tls_tcb *tcb = nullptr; |
| 517 |
# ifdef __HAVE___LWP_GETTCB_FAST |
| 518 |
tcb = (struct tls_tcb *)__lwp_gettcb_fast(); |
| 519 |
# elif defined(__HAVE___LWP_GETPRIVATE_FAST) |
| 520 |
tcb = (struct tls_tcb *)__lwp_getprivate_fast(); |
| 521 |
# endif |
| 522 |
return tcb; |
| 523 |
} |
| 524 |
|
| 525 |
uptr ThreadSelf() { return (uptr)ThreadSelfTlsTcb()->tcb_pthread; } |
| 526 |
|
| 527 |
int GetSizeFromHdr(struct dl_phdr_info *info, size_t size, void *data) { |
| 528 |
const Elf_Phdr *hdr = info->dlpi_phdr; |
| 529 |
const Elf_Phdr *last_hdr = hdr + info->dlpi_phnum; |
| 530 |
|
| 531 |
for (; hdr != last_hdr; ++hdr) { |
| 532 |
if (hdr->p_type == PT_TLS && info->dlpi_tls_modid == 1) { |
| 533 |
*(uptr *)data = hdr->p_memsz; |
| 534 |
break; |
| 535 |
} |
| 536 |
} |
| 537 |
return 0; |
| 538 |
} |
| 539 |
# endif // SANITIZER_NETBSD |
| 540 |
|
| 541 |
# if SANITIZER_ANDROID |
| 542 |
// Bionic provides this API since S. |
| 543 |
extern "C" SANITIZER_WEAK_ATTRIBUTE void __libc_get_static_tls_bounds(void **, |
| 544 |
void **); |
| 545 |
# endif |
| 546 |
|
| 547 |
# if !SANITIZER_GO |
| 548 |
static void GetTls(uptr *addr, uptr *size) { |
| 549 |
# if SANITIZER_ANDROID |
| 550 |
if (&__libc_get_static_tls_bounds) { |
| 551 |
void *start_addr; |
| 552 |
void *end_addr; |
| 553 |
__libc_get_static_tls_bounds(&start_addr, &end_addr); |
| 554 |
*addr = reinterpret_cast<uptr>(start_addr); |
| 555 |
*size = |
| 556 |
reinterpret_cast<uptr>(end_addr) - reinterpret_cast<uptr>(start_addr); |
| 557 |
} else { |
| 558 |
*addr = 0; |
| 559 |
*size = 0; |
| 560 |
} |
| 561 |
# elif SANITIZER_GLIBC && defined(__x86_64__) |
| 562 |
// For aarch64 and x86-64, use an O(1) approach which requires relatively |
| 563 |
// precise ThreadDescriptorSize. g_tls_size was initialized in InitTlsSize. |
| 564 |
# if SANITIZER_X32 |
| 565 |
asm("mov %%fs:8,%0" : "=r"(*addr)); |
| 566 |
# else |
| 567 |
asm("mov %%fs:16,%0" : "=r"(*addr)); |
| 568 |
# endif |
| 569 |
*size = g_tls_size; |
| 570 |
*addr -= *size; |
| 571 |
*addr += ThreadDescriptorSize(); |
| 572 |
# elif SANITIZER_GLIBC && defined(__aarch64__) |
| 573 |
*addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) - |
| 574 |
ThreadDescriptorSize(); |
| 575 |
*size = g_tls_size + ThreadDescriptorSize(); |
| 576 |
# elif SANITIZER_GLIBC && defined(__loongarch__) |
| 577 |
# ifdef __clang__ |
| 578 |
*addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) - |
| 579 |
ThreadDescriptorSize(); |
| 580 |
# else |
| 581 |
asm("or %0,$tp,$zero" : "=r"(*addr)); |
| 582 |
*addr -= ThreadDescriptorSize(); |
| 583 |
# endif |
| 584 |
*size = g_tls_size + ThreadDescriptorSize(); |
| 585 |
# elif SANITIZER_GLIBC && defined(__powerpc64__) |
| 586 |
// Workaround for glibc<2.25(?). 2.27 is known to not need this. |
| 587 |
uptr tp; |
| 588 |
asm("addi %0,13,-0x7000" : "=r"(tp)); |
| 589 |
const uptr pre_tcb_size = TlsPreTcbSize(); |
| 590 |
*addr = tp - pre_tcb_size; |
| 591 |
*size = g_tls_size + pre_tcb_size; |
| 592 |
# elif SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_SOLARIS |
| 593 |
uptr align; |
| 594 |
GetStaticTlsBoundary(addr, size, &align); |
| 595 |
# if defined(__x86_64__) || defined(__i386__) || defined(__s390__) || \ |
| 596 |
defined(__sparc__) |
| 597 |
if (SANITIZER_GLIBC) { |
| 598 |
# if defined(__x86_64__) || defined(__i386__) |
| 599 |
align = Max<uptr>(align, 64); |
| 600 |
# else |
| 601 |
align = Max<uptr>(align, 16); |
| 602 |
# endif |
| 603 |
} |
| 604 |
const uptr tp = RoundUpTo(*addr + *size, align); |
| 605 |
|
| 606 |
// lsan requires the range to additionally cover the static TLS surplus |
| 607 |
// (elf/dl-tls.c defines 1664). Otherwise there may be false positives for |
| 608 |
// allocations only referenced by tls in dynamically loaded modules. |
| 609 |
if (SANITIZER_GLIBC) |
| 610 |
*size += 1644; |
| 611 |
else if (SANITIZER_FREEBSD) |
| 612 |
*size += 128; // RTLD_STATIC_TLS_EXTRA |
| 613 |
|
| 614 |
// Extend the range to include the thread control block. On glibc, lsan needs |
| 615 |
// the range to include pthread::{specific_1stblock,specific} so that |
| 616 |
// allocations only referenced by pthread_setspecific can be scanned. This may |
| 617 |
// underestimate by at most TLS_TCB_ALIGN-1 bytes but it should be fine |
| 618 |
// because the number of bytes after pthread::specific is larger. |
| 619 |
*addr = tp - RoundUpTo(*size, align); |
| 620 |
*size = tp - *addr + ThreadDescriptorSize(); |
| 621 |
# else |
| 622 |
# if SANITIZER_GLIBC |
| 623 |
*size += 1664; |
| 624 |
# elif SANITIZER_FREEBSD |
| 625 |
*size += 128; // RTLD_STATIC_TLS_EXTRA |
| 626 |
# if defined(__mips__) || defined(__powerpc64__) || SANITIZER_RISCV64 |
| 627 |
const uptr pre_tcb_size = TlsPreTcbSize(); |
| 628 |
*addr -= pre_tcb_size; |
| 629 |
*size += pre_tcb_size; |
| 630 |
# else |
| 631 |
// arm and aarch64 reserve two words at TP, so this underestimates the range. |
| 632 |
// However, this is sufficient for the purpose of finding the pointers to |
| 633 |
// thread-specific data keys. |
| 634 |
const uptr tcb_size = ThreadDescriptorSize(); |
| 635 |
*addr -= tcb_size; |
| 636 |
*size += tcb_size; |
| 637 |
# endif |
| 638 |
# endif |
| 639 |
# endif |
| 640 |
# elif SANITIZER_NETBSD |
| 641 |
struct tls_tcb *const tcb = ThreadSelfTlsTcb(); |
| 642 |
*addr = 0; |
| 643 |
*size = 0; |
| 644 |
if (tcb != 0) { |
| 645 |
// Find size (p_memsz) of dlpi_tls_modid 1 (TLS block of the main program). |
| 646 |
// ld.elf_so hardcodes the index 1. |
| 647 |
dl_iterate_phdr(GetSizeFromHdr, size); |
| 648 |
|
| 649 |
if (*size != 0) { |
| 650 |
// The block has been found and tcb_dtv[1] contains the base address |
| 651 |
*addr = (uptr)tcb->tcb_dtv[1]; |
| 652 |
} |
| 653 |
} |
| 654 |
# else |
| 655 |
# error "Unknown OS" |
| 656 |
# endif |
| 657 |
} |
| 658 |
# endif |
| 659 |
|
| 660 |
# if !SANITIZER_GO |
| 661 |
uptr GetTlsSize() { |
| 662 |
# if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \ |
| 663 |
SANITIZER_SOLARIS |
| 664 |
uptr addr, size; |
| 665 |
GetTls(&addr, &size); |
| 666 |
return size; |
| 667 |
# else |
| 668 |
return 0; |
| 669 |
# endif |
| 670 |
} |
| 671 |
# endif |
| 672 |
|
| 673 |
void GetThreadStackAndTls(bool main, uptr *stk_begin, uptr *stk_end, |
| 674 |
uptr *tls_begin, uptr *tls_end) { |
| 675 |
# if SANITIZER_GO |
| 676 |
// Stub implementation for Go. |
| 677 |
*stk_begin = 0; |
| 678 |
*stk_end = 0; |
| 679 |
*tls_begin = 0; |
| 680 |
*tls_end = 0; |
| 681 |
# else |
| 682 |
uptr tls_addr = 0; |
| 683 |
uptr tls_size = 0; |
| 684 |
GetTls(&tls_addr, &tls_size); |
| 685 |
*tls_begin = tls_addr; |
| 686 |
*tls_end = tls_addr + tls_size; |
| 687 |
|
| 688 |
uptr stack_top, stack_bottom; |
| 689 |
GetThreadStackTopAndBottom(main, &stack_top, &stack_bottom); |
| 690 |
*stk_begin = stack_bottom; |
| 691 |
*stk_end = stack_top; |
| 692 |
|
| 693 |
if (!main) { |
| 694 |
// If stack and tls intersect, make them non-intersecting. |
| 695 |
if (*tls_begin > *stk_begin && *tls_begin < *stk_end) { |
| 696 |
if (*stk_end < *tls_end) |
| 697 |
*tls_end = *stk_end; |
| 698 |
*stk_end = *tls_begin; |
| 699 |
} |
| 700 |
} |
| 701 |
# endif |
| 702 |
} |
| 703 |
|
| 704 |
# if !SANITIZER_FREEBSD |
| 705 |
typedef ElfW(Phdr) Elf_Phdr; |
| 706 |
# endif |
| 707 |
|
| 708 |
struct DlIteratePhdrData { |
| 709 |
InternalMmapVectorNoCtor<LoadedModule> *modules; |
| 710 |
bool first; |
| 711 |
}; |
| 712 |
|
| 713 |
static int AddModuleSegments(const char *module_name, dl_phdr_info *info, |
| 714 |
InternalMmapVectorNoCtor<LoadedModule> *modules) { |
| 715 |
if (module_name[0] == '\0') |
| 716 |
return 0; |
| 717 |
LoadedModule cur_module; |
| 718 |
cur_module.set(module_name, info->dlpi_addr); |
| 719 |
for (int i = 0; i < (int)info->dlpi_phnum; i++) { |
| 720 |
const Elf_Phdr *phdr = &info->dlpi_phdr[i]; |
| 721 |
if (phdr->p_type == PT_LOAD) { |
| 722 |
uptr cur_beg = info->dlpi_addr + phdr->p_vaddr; |
| 723 |
uptr cur_end = cur_beg + phdr->p_memsz; |
| 724 |
bool executable = phdr->p_flags & PF_X; |
| 725 |
bool writable = phdr->p_flags & PF_W; |
| 726 |
cur_module.addAddressRange(cur_beg, cur_end, executable, writable); |
| 727 |
} else if (phdr->p_type == PT_NOTE) { |
| 728 |
# ifdef NT_GNU_BUILD_ID |
| 729 |
uptr off = 0; |
| 730 |
while (off + sizeof(ElfW(Nhdr)) < phdr->p_memsz) { |
| 731 |
auto *nhdr = reinterpret_cast<const ElfW(Nhdr) *>(info->dlpi_addr + |
| 732 |
phdr->p_vaddr + off); |
| 733 |
constexpr auto kGnuNamesz = 4; // "GNU" with NUL-byte. |
| 734 |
static_assert(kGnuNamesz % 4 == 0, "kGnuNameSize is aligned to 4."); |
| 735 |
if (nhdr->n_type == NT_GNU_BUILD_ID && nhdr->n_namesz == kGnuNamesz) { |
| 736 |
if (off + sizeof(ElfW(Nhdr)) + nhdr->n_namesz + nhdr->n_descsz > |
| 737 |
phdr->p_memsz) { |
| 738 |
// Something is very wrong, bail out instead of reading potentially |
| 739 |
// arbitrary memory. |
| 740 |
break; |
| 741 |
} |
| 742 |
const char *name = |
| 743 |
reinterpret_cast<const char *>(nhdr) + sizeof(*nhdr); |
| 744 |
if (internal_memcmp(name, "GNU", 3) == 0) { |
| 745 |
const char *value = reinterpret_cast<const char *>(nhdr) + |
| 746 |
sizeof(*nhdr) + kGnuNamesz; |
| 747 |
cur_module.setUuid(value, nhdr->n_descsz); |
| 748 |
break; |
| 749 |
} |
| 750 |
} |
| 751 |
off += sizeof(*nhdr) + RoundUpTo(nhdr->n_namesz, 4) + |
| 752 |
RoundUpTo(nhdr->n_descsz, 4); |
| 753 |
} |
| 754 |
# endif |
| 755 |
} |
| 756 |
} |
| 757 |
modules->push_back(cur_module); |
| 758 |
return 0; |
| 759 |
} |
| 760 |
|
| 761 |
static int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *arg) { |
| 762 |
DlIteratePhdrData *data = (DlIteratePhdrData *)arg; |
| 763 |
if (data->first) { |
| 764 |
InternalMmapVector<char> module_name(kMaxPathLength); |
| 765 |
data->first = false; |
| 766 |
// First module is the binary itself. |
| 767 |
ReadBinaryNameCached(module_name.data(), module_name.size()); |
| 768 |
return AddModuleSegments(module_name.data(), info, data->modules); |
| 769 |
} |
| 770 |
|
| 771 |
if (info->dlpi_name) |
| 772 |
return AddModuleSegments(info->dlpi_name, info, data->modules); |
| 773 |
|
| 774 |
return 0; |
| 775 |
} |
| 776 |
|
| 777 |
# if SANITIZER_ANDROID && __ANDROID_API__ < 21 |
| 778 |
extern "C" __attribute__((weak)) int dl_iterate_phdr( |
| 779 |
int (*)(struct dl_phdr_info *, size_t, void *), void *); |
| 780 |
# endif |
| 781 |
|
| 782 |
static bool requiresProcmaps() { |
| 783 |
# if SANITIZER_ANDROID && __ANDROID_API__ <= 22 |
| 784 |
// Fall back to /proc/maps if dl_iterate_phdr is unavailable or broken. |
| 785 |
// The runtime check allows the same library to work with |
| 786 |
// both K and L (and future) Android releases. |
| 787 |
return AndroidGetApiLevel() <= ANDROID_LOLLIPOP_MR1; |
| 788 |
# else |
| 789 |
return false; |
| 790 |
# endif |
| 791 |
} |
| 792 |
|
| 793 |
static void procmapsInit(InternalMmapVectorNoCtor<LoadedModule> *modules) { |
| 794 |
MemoryMappingLayout memory_mapping(/*cache_enabled*/ true); |
| 795 |
memory_mapping.DumpListOfModules(modules); |
| 796 |
} |
| 797 |
|
| 798 |
void ListOfModules::init() { |
| 799 |
clearOrInit(); |
| 800 |
if (requiresProcmaps()) { |
| 801 |
procmapsInit(&modules_); |
| 802 |
} else { |
| 803 |
DlIteratePhdrData data = {&modules_, true}; |
| 804 |
dl_iterate_phdr(dl_iterate_phdr_cb, &data); |
| 805 |
} |
| 806 |
} |
| 807 |
|
| 808 |
// When a custom loader is used, dl_iterate_phdr may not contain the full |
| 809 |
// list of modules. Allow callers to fall back to using procmaps. |
| 810 |
void ListOfModules::fallbackInit() { |
| 811 |
if (!requiresProcmaps()) { |
| 812 |
clearOrInit(); |
| 813 |
procmapsInit(&modules_); |
| 814 |
} else { |
| 815 |
clear(); |
| 816 |
} |
| 817 |
} |
| 818 |
|
| 819 |
// getrusage does not give us the current RSS, only the max RSS. |
| 820 |
// Still, this is better than nothing if /proc/self/statm is not available |
| 821 |
// for some reason, e.g. due to a sandbox. |
| 822 |
static uptr GetRSSFromGetrusage() { |
| 823 |
struct rusage usage; |
| 824 |
if (getrusage(RUSAGE_SELF, &usage)) // Failed, probably due to a sandbox. |
| 825 |
return 0; |
| 826 |
return usage.ru_maxrss << 10; // ru_maxrss is in Kb. |
| 827 |
} |
| 828 |
|
| 829 |
uptr GetRSS() { |
| 830 |
if (!common_flags()->can_use_proc_maps_statm) |
| 831 |
return GetRSSFromGetrusage(); |
| 832 |
fd_t fd = OpenFile("/proc/self/statm", RdOnly); |
| 833 |
if (fd == kInvalidFd) |
| 834 |
return GetRSSFromGetrusage(); |
| 835 |
char buf[64]; |
| 836 |
uptr len = internal_read(fd, buf, sizeof(buf) - 1); |
| 837 |
internal_close(fd); |
| 838 |
if ((sptr)len <= 0) |
| 839 |
return 0; |
| 840 |
buf[len] = 0; |
| 841 |
// The format of the file is: |
| 842 |
// 1084 89 69 11 0 79 0 |
| 843 |
// We need the second number which is RSS in pages. |
| 844 |
char *pos = buf; |
| 845 |
// Skip the first number. |
| 846 |
while (*pos >= '0' && *pos <= '9') pos++; |
| 847 |
// Skip whitespaces. |
| 848 |
while (!(*pos >= '0' && *pos <= '9') && *pos != 0) pos++; |
| 849 |
// Read the number. |
| 850 |
uptr rss = 0; |
| 851 |
while (*pos >= '0' && *pos <= '9') rss = rss * 10 + *pos++ - '0'; |
| 852 |
return rss * GetPageSizeCached(); |
| 853 |
} |
| 854 |
|
| 855 |
// sysconf(_SC_NPROCESSORS_{CONF,ONLN}) cannot be used on most platforms as |
| 856 |
// they allocate memory. |
| 857 |
u32 GetNumberOfCPUs() { |
| 858 |
# if SANITIZER_FREEBSD || SANITIZER_NETBSD |
| 859 |
u32 ncpu; |
| 860 |
int req[2]; |
| 861 |
uptr len = sizeof(ncpu); |
| 862 |
req[0] = CTL_HW; |
| 863 |
req[1] = HW_NCPU; |
| 864 |
CHECK_EQ(internal_sysctl(req, 2, &ncpu, &len, NULL, 0), 0); |
| 865 |
return ncpu; |
| 866 |
# elif SANITIZER_ANDROID && !defined(CPU_COUNT) && !defined(__aarch64__) |
| 867 |
// Fall back to /sys/devices/system/cpu on Android when cpu_set_t doesn't |
| 868 |
// exist in sched.h. That is the case for toolchains generated with older |
| 869 |
// NDKs. |
| 870 |
// This code doesn't work on AArch64 because internal_getdents makes use of |
| 871 |
// the 64bit getdents syscall, but cpu_set_t seems to always exist on AArch64. |
| 872 |
uptr fd = internal_open("/sys/devices/system/cpu", O_RDONLY | O_DIRECTORY); |
| 873 |
if (internal_iserror(fd)) |
| 874 |
return 0; |
| 875 |
InternalMmapVector<u8> buffer(4096); |
| 876 |
uptr bytes_read = buffer.size(); |
| 877 |
uptr n_cpus = 0; |
| 878 |
u8 *d_type; |
| 879 |
struct linux_dirent *entry = (struct linux_dirent *)&buffer[bytes_read]; |
| 880 |
while (true) { |
| 881 |
if ((u8 *)entry >= &buffer[bytes_read]) { |
| 882 |
bytes_read = internal_getdents(fd, (struct linux_dirent *)buffer.data(), |
| 883 |
buffer.size()); |
| 884 |
if (internal_iserror(bytes_read) || !bytes_read) |
| 885 |
break; |
| 886 |
entry = (struct linux_dirent *)buffer.data(); |
| 887 |
} |
| 888 |
d_type = (u8 *)entry + entry->d_reclen - 1; |
| 889 |
if (d_type >= &buffer[bytes_read] || |
| 890 |
(u8 *)&entry->d_name[3] >= &buffer[bytes_read]) |
| 891 |
break; |
| 892 |
if (entry->d_ino != 0 && *d_type == DT_DIR) { |
| 893 |
if (entry->d_name[0] == 'c' && entry->d_name[1] == 'p' && |
| 894 |
entry->d_name[2] == 'u' && entry->d_name[3] >= '0' && |
| 895 |
entry->d_name[3] <= '9') |
| 896 |
n_cpus++; |
| 897 |
} |
| 898 |
entry = (struct linux_dirent *)(((u8 *)entry) + entry->d_reclen); |
| 899 |
} |
| 900 |
internal_close(fd); |
| 901 |
return n_cpus; |
| 902 |
# elif SANITIZER_SOLARIS |
| 903 |
return sysconf(_SC_NPROCESSORS_ONLN); |
| 904 |
# else |
| 905 |
# if defined(CPU_COUNT) |
| 906 |
cpu_set_t CPUs; |
| 907 |
CHECK_EQ(sched_getaffinity(0, sizeof(cpu_set_t), &CPUs), 0); |
| 908 |
return CPU_COUNT(&CPUs); |
| 909 |
# else |
| 910 |
return 1; |
| 911 |
# endif |
| 912 |
# endif |
| 913 |
} |
| 914 |
|
| 915 |
# if SANITIZER_LINUX |
| 916 |
|
| 917 |
# if SANITIZER_ANDROID |
| 918 |
static atomic_uint8_t android_log_initialized; |
| 919 |
|
| 920 |
void AndroidLogInit() { |
| 921 |
openlog(GetProcessName(), 0, LOG_USER); |
| 922 |
atomic_store(&android_log_initialized, 1, memory_order_release); |
| 923 |
} |
| 924 |
|
| 925 |
static bool ShouldLogAfterPrintf() { |
| 926 |
return atomic_load(&android_log_initialized, memory_order_acquire); |
| 927 |
} |
| 928 |
|
| 929 |
extern "C" SANITIZER_WEAK_ATTRIBUTE int async_safe_write_log(int pri, |
| 930 |
const char *tag, |
| 931 |
const char *msg); |
| 932 |
extern "C" SANITIZER_WEAK_ATTRIBUTE int __android_log_write(int prio, |
| 933 |
const char *tag, |
| 934 |
const char *msg); |
| 935 |
|
| 936 |
// ANDROID_LOG_INFO is 4, but can't be resolved at runtime. |
| 937 |
# define SANITIZER_ANDROID_LOG_INFO 4 |
| 938 |
|
| 939 |
// async_safe_write_log is a new public version of __libc_write_log that is |
| 940 |
// used behind syslog. It is preferable to syslog as it will not do any dynamic |
| 941 |
// memory allocation or formatting. |
| 942 |
// If the function is not available, syslog is preferred for L+ (it was broken |
| 943 |
// pre-L) as __android_log_write triggers a racey behavior with the strncpy |
| 944 |
// interceptor. Fallback to __android_log_write pre-L. |
| 945 |
void WriteOneLineToSyslog(const char *s) { |
| 946 |
if (&async_safe_write_log) { |
| 947 |
async_safe_write_log(SANITIZER_ANDROID_LOG_INFO, GetProcessName(), s); |
| 948 |
} else if (AndroidGetApiLevel() > ANDROID_KITKAT) { |
| 949 |
syslog(LOG_INFO, "%s", s); |
| 950 |
} else { |
| 951 |
CHECK(&__android_log_write); |
| 952 |
__android_log_write(SANITIZER_ANDROID_LOG_INFO, nullptr, s); |
| 953 |
} |
| 954 |
} |
| 955 |
|
| 956 |
extern "C" SANITIZER_WEAK_ATTRIBUTE void android_set_abort_message( |
| 957 |
const char *); |
| 958 |
|
| 959 |
void SetAbortMessage(const char *str) { |
| 960 |
if (&android_set_abort_message) |
| 961 |
android_set_abort_message(str); |
| 962 |
} |
| 963 |
# else |
| 964 |
void AndroidLogInit() {} |
| 965 |
|
| 966 |
static bool ShouldLogAfterPrintf() { return true; } |
| 967 |
|
| 968 |
void WriteOneLineToSyslog(const char *s) { syslog(LOG_INFO, "%s", s); } |
| 969 |
|
| 970 |
void SetAbortMessage(const char *str) {} |
| 971 |
# endif // SANITIZER_ANDROID |
| 972 |
|
| 973 |
void LogMessageOnPrintf(const char *str) { |
| 974 |
if (common_flags()->log_to_syslog && ShouldLogAfterPrintf()) |
| 975 |
WriteToSyslog(str); |
| 976 |
} |
| 977 |
|
| 978 |
# endif // SANITIZER_LINUX |
| 979 |
|
| 980 |
# if SANITIZER_GLIBC && !SANITIZER_GO |
| 981 |
// glibc crashes when using clock_gettime from a preinit_array function as the |
| 982 |
// vDSO function pointers haven't been initialized yet. __progname is |
| 983 |
// initialized after the vDSO function pointers, so if it exists, is not null |
| 984 |
// and is not empty, we can use clock_gettime. |
| 985 |
extern "C" SANITIZER_WEAK_ATTRIBUTE char *__progname; |
| 986 |
inline bool CanUseVDSO() { return &__progname && __progname && *__progname; } |
| 987 |
|
| 988 |
// MonotonicNanoTime is a timing function that can leverage the vDSO by calling |
| 989 |
// clock_gettime. real_clock_gettime only exists if clock_gettime is |
| 990 |
// intercepted, so define it weakly and use it if available. |
| 991 |
extern "C" SANITIZER_WEAK_ATTRIBUTE int real_clock_gettime(u32 clk_id, |
| 992 |
void *tp); |
| 993 |
u64 MonotonicNanoTime() { |
| 994 |
timespec ts; |
| 995 |
if (CanUseVDSO()) { |
| 996 |
if (&real_clock_gettime) |
| 997 |
real_clock_gettime(CLOCK_MONOTONIC, &ts); |
| 998 |
else |
| 999 |
clock_gettime(CLOCK_MONOTONIC, &ts); |
| 1000 |
} else { |
| 1001 |
internal_clock_gettime(CLOCK_MONOTONIC, &ts); |
| 1002 |
} |
| 1003 |
return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec; |
| 1004 |
} |
| 1005 |
# else |
| 1006 |
// Non-glibc & Go always use the regular function. |
| 1007 |
u64 MonotonicNanoTime() { |
| 1008 |
timespec ts; |
| 1009 |
clock_gettime(CLOCK_MONOTONIC, &ts); |
| 1010 |
return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec; |
| 1011 |
} |
| 1012 |
# endif // SANITIZER_GLIBC && !SANITIZER_GO |
| 1013 |
|
| 1014 |
void ReExec() { |
| 1015 |
const char *pathname = "/proc/self/exe"; |
| 1016 |
|
| 1017 |
# if SANITIZER_FREEBSD |
| 1018 |
for (const auto *aux = __elf_aux_vector; aux->a_type != AT_NULL; aux++) { |
| 1019 |
if (aux->a_type == AT_EXECPATH) { |
| 1020 |
pathname = static_cast<const char *>(aux->a_un.a_ptr); |
| 1021 |
break; |
| 1022 |
} |
| 1023 |
} |
| 1024 |
# elif SANITIZER_NETBSD |
| 1025 |
static const int name[] = { |
| 1026 |
CTL_KERN, |
| 1027 |
KERN_PROC_ARGS, |
| 1028 |
-1, |
| 1029 |
KERN_PROC_PATHNAME, |
| 1030 |
}; |
| 1031 |
char path[400]; |
| 1032 |
uptr len; |
| 1033 |
|
| 1034 |
len = sizeof(path); |
| 1035 |
if (internal_sysctl(name, ARRAY_SIZE(name), path, &len, NULL, 0) != -1) |
| 1036 |
pathname = path; |
| 1037 |
# elif SANITIZER_SOLARIS |
| 1038 |
pathname = getexecname(); |
| 1039 |
CHECK_NE(pathname, NULL); |
| 1040 |
# elif SANITIZER_USE_GETAUXVAL |
| 1041 |
// Calling execve with /proc/self/exe sets that as $EXEC_ORIGIN. Binaries that |
| 1042 |
// rely on that will fail to load shared libraries. Query AT_EXECFN instead. |
| 1043 |
pathname = reinterpret_cast<const char *>(getauxval(AT_EXECFN)); |
| 1044 |
# endif |
| 1045 |
|
| 1046 |
uptr rv = internal_execve(pathname, GetArgv(), GetEnviron()); |
| 1047 |
int rverrno; |
| 1048 |
CHECK_EQ(internal_iserror(rv, &rverrno), true); |
| 1049 |
Printf("execve failed, errno %d\n", rverrno); |
| 1050 |
Die(); |
| 1051 |
} |
| 1052 |
|
| 1053 |
void UnmapFromTo(uptr from, uptr to) { |
| 1054 |
if (to == from) |
| 1055 |
return; |
| 1056 |
CHECK(to >= from); |
| 1057 |
uptr res = internal_munmap(reinterpret_cast<void *>(from), to - from); |
| 1058 |
if (UNLIKELY(internal_iserror(res))) { |
| 1059 |
Report("ERROR: %s failed to unmap 0x%zx (%zd) bytes at address %p\n", |
| 1060 |
SanitizerToolName, to - from, to - from, (void *)from); |
| 1061 |
CHECK("unable to unmap" && 0); |
| 1062 |
} |
| 1063 |
} |
| 1064 |
|
| 1065 |
uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale, |
| 1066 |
uptr min_shadow_base_alignment, UNUSED uptr &high_mem_end, |
| 1067 |
uptr granularity) { |
| 1068 |
const uptr alignment = |
| 1069 |
Max<uptr>(granularity << shadow_scale, 1ULL << min_shadow_base_alignment); |
| 1070 |
const uptr left_padding = |
| 1071 |
Max<uptr>(granularity, 1ULL << min_shadow_base_alignment); |
| 1072 |
|
| 1073 |
const uptr shadow_size = RoundUpTo(shadow_size_bytes, granularity); |
| 1074 |
const uptr map_size = shadow_size + left_padding + alignment; |
| 1075 |
|
| 1076 |
const uptr map_start = (uptr)MmapNoAccess(map_size); |
| 1077 |
CHECK_NE(map_start, ~(uptr)0); |
| 1078 |
|
| 1079 |
const uptr shadow_start = RoundUpTo(map_start + left_padding, alignment); |
| 1080 |
|
| 1081 |
UnmapFromTo(map_start, shadow_start - left_padding); |
| 1082 |
UnmapFromTo(shadow_start + shadow_size, map_start + map_size); |
| 1083 |
|
| 1084 |
return shadow_start; |
| 1085 |
} |
| 1086 |
|
| 1087 |
static uptr MmapSharedNoReserve(uptr addr, uptr size) { |
| 1088 |
return internal_mmap( |
| 1089 |
reinterpret_cast<void *>(addr), size, PROT_READ | PROT_WRITE, |
| 1090 |
MAP_FIXED | MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); |
| 1091 |
} |
| 1092 |
|
| 1093 |
static uptr MremapCreateAlias(uptr base_addr, uptr alias_addr, |
| 1094 |
uptr alias_size) { |
| 1095 |
# if SANITIZER_LINUX |
| 1096 |
return internal_mremap(reinterpret_cast<void *>(base_addr), 0, alias_size, |
| 1097 |
MREMAP_MAYMOVE | MREMAP_FIXED, |
| 1098 |
reinterpret_cast<void *>(alias_addr)); |
| 1099 |
# else |
| 1100 |
CHECK(false && "mremap is not supported outside of Linux"); |
| 1101 |
return 0; |
| 1102 |
# endif |
| 1103 |
} |
| 1104 |
|
| 1105 |
static void CreateAliases(uptr start_addr, uptr alias_size, uptr num_aliases) { |
| 1106 |
uptr total_size = alias_size * num_aliases; |
| 1107 |
uptr mapped = MmapSharedNoReserve(start_addr, total_size); |
| 1108 |
CHECK_EQ(mapped, start_addr); |
| 1109 |
|
| 1110 |
for (uptr i = 1; i < num_aliases; ++i) { |
| 1111 |
uptr alias_addr = start_addr + i * alias_size; |
| 1112 |
CHECK_EQ(MremapCreateAlias(start_addr, alias_addr, alias_size), alias_addr); |
| 1113 |
} |
| 1114 |
} |
| 1115 |
|
| 1116 |
uptr MapDynamicShadowAndAliases(uptr shadow_size, uptr alias_size, |
| 1117 |
uptr num_aliases, uptr ring_buffer_size) { |
| 1118 |
CHECK_EQ(alias_size & (alias_size - 1), 0); |
| 1119 |
CHECK_EQ(num_aliases & (num_aliases - 1), 0); |
| 1120 |
CHECK_EQ(ring_buffer_size & (ring_buffer_size - 1), 0); |
| 1121 |
|
| 1122 |
const uptr granularity = GetMmapGranularity(); |
| 1123 |
shadow_size = RoundUpTo(shadow_size, granularity); |
| 1124 |
CHECK_EQ(shadow_size & (shadow_size - 1), 0); |
| 1125 |
|
| 1126 |
const uptr alias_region_size = alias_size * num_aliases; |
| 1127 |
const uptr alignment = |
| 1128 |
2 * Max(Max(shadow_size, alias_region_size), ring_buffer_size); |
| 1129 |
const uptr left_padding = ring_buffer_size; |
| 1130 |
|
| 1131 |
const uptr right_size = alignment; |
| 1132 |
const uptr map_size = left_padding + 2 * alignment; |
| 1133 |
|
| 1134 |
const uptr map_start = reinterpret_cast<uptr>(MmapNoAccess(map_size)); |
| 1135 |
CHECK_NE(map_start, static_cast<uptr>(-1)); |
| 1136 |
const uptr right_start = RoundUpTo(map_start + left_padding, alignment); |
| 1137 |
|
| 1138 |
UnmapFromTo(map_start, right_start - left_padding); |
| 1139 |
UnmapFromTo(right_start + right_size, map_start + map_size); |
| 1140 |
|
| 1141 |
CreateAliases(right_start + right_size / 2, alias_size, num_aliases); |
| 1142 |
|
| 1143 |
return right_start; |
| 1144 |
} |
| 1145 |
|
| 1146 |
void InitializePlatformCommonFlags(CommonFlags *cf) { |
| 1147 |
# if SANITIZER_ANDROID |
| 1148 |
if (&__libc_get_static_tls_bounds == nullptr) |
| 1149 |
cf->detect_leaks = false; |
| 1150 |
# endif |
| 1151 |
} |
| 1152 |
|
| 1153 |
} // namespace __sanitizer |
| 1154 |
|
| 1155 |
#endif |