| 1 |
/* |
| 2 |
* Copyright (c) 2000-2005 by Hewlett-Packard Company. All rights reserved. |
| 3 |
* |
| 4 |
* THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED |
| 5 |
* OR IMPLIED. ANY USE IS AT YOUR OWN RISK. |
| 6 |
* |
| 7 |
* Permission is hereby granted to use or copy this program |
| 8 |
* for any purpose, provided the above notices are retained on all copies. |
| 9 |
* Permission to modify the code and to distribute modified code is granted, |
| 10 |
* provided the above notices are retained, and a notice that the code was |
| 11 |
* modified is included with the above copyright notice. |
| 12 |
*/ |
| 13 |
|
| 14 |
#include "private/gc_priv.h" |
| 15 |
|
| 16 |
#if defined(THREAD_LOCAL_ALLOC) |
| 17 |
|
| 18 |
#ifndef THREADS |
| 19 |
# error "invalid config - THREAD_LOCAL_ALLOC requires GC_THREADS" |
| 20 |
#endif |
| 21 |
|
| 22 |
#include "private/thread_local_alloc.h" |
| 23 |
|
| 24 |
#include <stdlib.h> |
| 25 |
|
| 26 |
#if defined(USE_COMPILER_TLS) |
| 27 |
__thread GC_ATTR_TLS_FAST |
| 28 |
#elif defined(USE_WIN32_COMPILER_TLS) |
| 29 |
__declspec(thread) GC_ATTR_TLS_FAST |
| 30 |
#endif |
| 31 |
GC_key_t GC_thread_key; |
| 32 |
|
| 33 |
static GC_bool keys_initialized; |
| 34 |
|
| 35 |
/* Return a single nonempty freelist fl to the global one pointed to */ |
| 36 |
/* by gfl. */ |
| 37 |
|
| 38 |
static void return_single_freelist(void *fl, void **gfl) |
| 39 |
{ |
| 40 |
if (*gfl == 0) { |
| 41 |
*gfl = fl; |
| 42 |
} else { |
| 43 |
void *q, **qptr; |
| 44 |
|
| 45 |
GC_ASSERT(GC_size(fl) == GC_size(*gfl)); |
| 46 |
/* Concatenate: */ |
| 47 |
qptr = &(obj_link(fl)); |
| 48 |
while ((word)(q = *qptr) >= HBLKSIZE) |
| 49 |
qptr = &(obj_link(q)); |
| 50 |
GC_ASSERT(0 == q); |
| 51 |
*qptr = *gfl; |
| 52 |
*gfl = fl; |
| 53 |
} |
| 54 |
} |
| 55 |
|
| 56 |
/* Recover the contents of the freelist array fl into the global one gfl.*/ |
| 57 |
/* We hold the allocator lock. */ |
| 58 |
static void return_freelists(void **fl, void **gfl) |
| 59 |
{ |
| 60 |
int i; |
| 61 |
|
| 62 |
for (i = 1; i < TINY_FREELISTS; ++i) { |
| 63 |
if ((word)(fl[i]) >= HBLKSIZE) { |
| 64 |
return_single_freelist(fl[i], &gfl[i]); |
| 65 |
} |
| 66 |
/* Clear fl[i], since the thread structure may hang around. */ |
| 67 |
/* Do it in a way that is likely to trap if we access it. */ |
| 68 |
fl[i] = (ptr_t)HBLKSIZE; |
| 69 |
} |
| 70 |
/* The 0 granule freelist really contains 1 granule objects. */ |
| 71 |
# ifdef GC_GCJ_SUPPORT |
| 72 |
if (fl[0] == ERROR_FL) return; |
| 73 |
# endif |
| 74 |
if ((word)(fl[0]) >= HBLKSIZE) { |
| 75 |
return_single_freelist(fl[0], &gfl[1]); |
| 76 |
} |
| 77 |
} |
| 78 |
|
| 79 |
#ifdef USE_PTHREAD_SPECIFIC |
| 80 |
/* Re-set the TLS value on thread cleanup to allow thread-local */ |
| 81 |
/* allocations to happen in the TLS destructors. */ |
| 82 |
/* GC_unregister_my_thread (and similar routines) will finally set */ |
| 83 |
/* the GC_thread_key to NULL preventing this destructor from being */ |
| 84 |
/* called repeatedly. */ |
| 85 |
static void reset_thread_key(void* v) { |
| 86 |
pthread_setspecific(GC_thread_key, v); |
| 87 |
} |
| 88 |
#else |
| 89 |
# define reset_thread_key 0 |
| 90 |
#endif |
| 91 |
|
| 92 |
/* Each thread structure must be initialized. */ |
| 93 |
/* This call must be made from the new thread. */ |
| 94 |
GC_INNER void GC_init_thread_local(GC_tlfs p) |
| 95 |
{ |
| 96 |
int i, j, res; |
| 97 |
|
| 98 |
GC_ASSERT(I_HOLD_LOCK()); |
| 99 |
if (!EXPECT(keys_initialized, TRUE)) { |
| 100 |
# ifdef USE_CUSTOM_SPECIFIC |
| 101 |
/* Ensure proper alignment of a "pushed" GC symbol. */ |
| 102 |
GC_ASSERT((word)&GC_thread_key % sizeof(word) == 0); |
| 103 |
# endif |
| 104 |
res = GC_key_create(&GC_thread_key, reset_thread_key); |
| 105 |
if (COVERT_DATAFLOW(res) != 0) { |
| 106 |
ABORT("Failed to create key for local allocator"); |
| 107 |
} |
| 108 |
keys_initialized = TRUE; |
| 109 |
} |
| 110 |
res = GC_setspecific(GC_thread_key, p); |
| 111 |
if (COVERT_DATAFLOW(res) != 0) { |
| 112 |
ABORT("Failed to set thread specific allocation pointers"); |
| 113 |
} |
| 114 |
for (j = 0; j < TINY_FREELISTS; ++j) { |
| 115 |
for (i = 0; i < THREAD_FREELISTS_KINDS; ++i) { |
| 116 |
p -> _freelists[i][j] = (void *)(word)1; |
| 117 |
} |
| 118 |
# ifdef GC_GCJ_SUPPORT |
| 119 |
p -> gcj_freelists[j] = (void *)(word)1; |
| 120 |
# endif |
| 121 |
} |
| 122 |
/* The size 0 free lists are handled like the regular free lists, */ |
| 123 |
/* to ensure that the explicit deallocation works. However, */ |
| 124 |
/* allocation of a size 0 "gcj" object is always an error. */ |
| 125 |
# ifdef GC_GCJ_SUPPORT |
| 126 |
p -> gcj_freelists[0] = ERROR_FL; |
| 127 |
# endif |
| 128 |
} |
| 129 |
|
| 130 |
/* We hold the allocator lock. */ |
| 131 |
GC_INNER void GC_destroy_thread_local(GC_tlfs p) |
| 132 |
{ |
| 133 |
int k; |
| 134 |
|
| 135 |
/* We currently only do this from the thread itself. */ |
| 136 |
GC_STATIC_ASSERT(THREAD_FREELISTS_KINDS <= MAXOBJKINDS); |
| 137 |
for (k = 0; k < THREAD_FREELISTS_KINDS; ++k) { |
| 138 |
if (k == (int)GC_n_kinds) |
| 139 |
break; /* kind is not created */ |
| 140 |
return_freelists(p -> _freelists[k], GC_obj_kinds[k].ok_freelist); |
| 141 |
} |
| 142 |
# ifdef GC_GCJ_SUPPORT |
| 143 |
return_freelists(p -> gcj_freelists, (void **)GC_gcjobjfreelist); |
| 144 |
# endif |
| 145 |
} |
| 146 |
|
| 147 |
GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_kind(size_t bytes, int kind) |
| 148 |
{ |
| 149 |
size_t granules; |
| 150 |
void *tsd; |
| 151 |
void *result; |
| 152 |
|
| 153 |
# if MAXOBJKINDS > THREAD_FREELISTS_KINDS |
| 154 |
if (EXPECT(kind >= THREAD_FREELISTS_KINDS, FALSE)) { |
| 155 |
return GC_malloc_kind_global(bytes, kind); |
| 156 |
} |
| 157 |
# endif |
| 158 |
# if !defined(USE_PTHREAD_SPECIFIC) && !defined(USE_WIN32_SPECIFIC) |
| 159 |
{ |
| 160 |
GC_key_t k = GC_thread_key; |
| 161 |
|
| 162 |
if (EXPECT(0 == k, FALSE)) { |
| 163 |
/* We haven't yet run GC_init_parallel. That means */ |
| 164 |
/* we also aren't locking, so this is fairly cheap. */ |
| 165 |
return GC_malloc_kind_global(bytes, kind); |
| 166 |
} |
| 167 |
tsd = GC_getspecific(k); |
| 168 |
} |
| 169 |
# else |
| 170 |
if (!EXPECT(keys_initialized, TRUE)) |
| 171 |
return GC_malloc_kind_global(bytes, kind); |
| 172 |
tsd = GC_getspecific(GC_thread_key); |
| 173 |
# endif |
| 174 |
# if !defined(USE_COMPILER_TLS) && !defined(USE_WIN32_COMPILER_TLS) |
| 175 |
if (EXPECT(0 == tsd, FALSE)) { |
| 176 |
return GC_malloc_kind_global(bytes, kind); |
| 177 |
} |
| 178 |
# endif |
| 179 |
GC_ASSERT(GC_is_initialized); |
| 180 |
GC_ASSERT(GC_is_thread_tsd_valid(tsd)); |
| 181 |
granules = ROUNDED_UP_GRANULES(bytes); |
| 182 |
# if defined(CPPCHECK) |
| 183 |
# define MALLOC_KIND_PTRFREE_INIT (void*)1 |
| 184 |
# else |
| 185 |
# define MALLOC_KIND_PTRFREE_INIT NULL |
| 186 |
# endif |
| 187 |
GC_FAST_MALLOC_GRANS(result, granules, |
| 188 |
((GC_tlfs)tsd) -> _freelists[kind], DIRECT_GRANULES, |
| 189 |
kind, GC_malloc_kind_global(bytes, kind), |
| 190 |
(void)(kind == PTRFREE ? MALLOC_KIND_PTRFREE_INIT |
| 191 |
: (obj_link(result) = 0))); |
| 192 |
# ifdef LOG_ALLOCS |
| 193 |
GC_log_printf("GC_malloc_kind(%lu, %d) returned %p, recent GC #%lu\n", |
| 194 |
(unsigned long)bytes, kind, result, |
| 195 |
(unsigned long)GC_gc_no); |
| 196 |
# endif |
| 197 |
return result; |
| 198 |
} |
| 199 |
|
| 200 |
#ifdef GC_GCJ_SUPPORT |
| 201 |
|
| 202 |
# include "gc_gcj.h" |
| 203 |
|
| 204 |
/* Gcj-style allocation without locks is extremely tricky. The */ |
| 205 |
/* fundamental issue is that we may end up marking a free list, which */ |
| 206 |
/* has freelist links instead of "vtable" pointers. That is usually */ |
| 207 |
/* OK, since the next object on the free list will be cleared, and */ |
| 208 |
/* will thus be interpreted as containing a zero descriptor. That's */ |
| 209 |
/* fine if the object has not yet been initialized. But there are */ |
| 210 |
/* interesting potential races. */ |
| 211 |
/* In the case of incremental collection, this seems hopeless, since */ |
| 212 |
/* the marker may run asynchronously, and may pick up the pointer to */ |
| 213 |
/* the next freelist entry (which it thinks is a vtable pointer), get */ |
| 214 |
/* suspended for a while, and then see an allocated object instead */ |
| 215 |
/* of the vtable. This may be avoidable with either a handshake with */ |
| 216 |
/* the collector or, probably more easily, by moving the free list */ |
| 217 |
/* links to the second word of each object. The latter isn't a */ |
| 218 |
/* universal win, since on architecture like Itanium, nonzero offsets */ |
| 219 |
/* are not necessarily free. And there may be cache fill order issues. */ |
| 220 |
/* For now, we punt with incremental GC. This probably means that */ |
| 221 |
/* incremental GC should be enabled before we fork a second thread. */ |
| 222 |
/* Unlike the other thread local allocation calls, we assume that the */ |
| 223 |
/* collector has been explicitly initialized. */ |
| 224 |
GC_API GC_ATTR_MALLOC void * GC_CALL GC_gcj_malloc(size_t bytes, |
| 225 |
void * ptr_to_struct_containing_descr) |
| 226 |
{ |
| 227 |
if (EXPECT(GC_incremental, FALSE)) { |
| 228 |
return GC_core_gcj_malloc(bytes, ptr_to_struct_containing_descr); |
| 229 |
} else { |
| 230 |
size_t granules = ROUNDED_UP_GRANULES(bytes); |
| 231 |
void *result; |
| 232 |
void **tiny_fl; |
| 233 |
|
| 234 |
GC_ASSERT(GC_gcjobjfreelist != NULL); |
| 235 |
tiny_fl = ((GC_tlfs)GC_getspecific(GC_thread_key))->gcj_freelists; |
| 236 |
GC_FAST_MALLOC_GRANS(result, granules, tiny_fl, DIRECT_GRANULES, |
| 237 |
GC_gcj_kind, |
| 238 |
GC_core_gcj_malloc(bytes, |
| 239 |
ptr_to_struct_containing_descr), |
| 240 |
{AO_compiler_barrier(); |
| 241 |
*(void **)result = ptr_to_struct_containing_descr;}); |
| 242 |
/* This forces the initialization of the "method ptr". */ |
| 243 |
/* This is necessary to ensure some very subtle properties */ |
| 244 |
/* required if a GC is run in the middle of such an allocation. */ |
| 245 |
/* Here we implicitly also assume atomicity for the free list. */ |
| 246 |
/* and method pointer assignments. */ |
| 247 |
/* We must update the freelist before we store the pointer. */ |
| 248 |
/* Otherwise a GC at this point would see a corrupted */ |
| 249 |
/* free list. */ |
| 250 |
/* A real memory barrier is not needed, since the */ |
| 251 |
/* action of stopping this thread will cause prior writes */ |
| 252 |
/* to complete. */ |
| 253 |
/* We assert that any concurrent marker will stop us. */ |
| 254 |
/* Thus it is impossible for a mark procedure to see the */ |
| 255 |
/* allocation of the next object, but to see this object */ |
| 256 |
/* still containing a free list pointer. Otherwise the */ |
| 257 |
/* marker, by misinterpreting the freelist link as a vtable */ |
| 258 |
/* pointer, might find a random "mark descriptor" in the next */ |
| 259 |
/* object. */ |
| 260 |
return result; |
| 261 |
} |
| 262 |
} |
| 263 |
|
| 264 |
#endif /* GC_GCJ_SUPPORT */ |
| 265 |
|
| 266 |
/* The thread support layer must arrange to mark thread-local */ |
| 267 |
/* free lists explicitly, since the link field is often */ |
| 268 |
/* invisible to the marker. It knows how to find all threads; */ |
| 269 |
/* we take care of an individual thread freelist structure. */ |
| 270 |
GC_INNER void GC_mark_thread_local_fls_for(GC_tlfs p) |
| 271 |
{ |
| 272 |
ptr_t q; |
| 273 |
int i, j; |
| 274 |
|
| 275 |
for (j = 0; j < TINY_FREELISTS; ++j) { |
| 276 |
for (i = 0; i < THREAD_FREELISTS_KINDS; ++i) { |
| 277 |
/* Load the pointer atomically as it might be updated */ |
| 278 |
/* concurrently by GC_FAST_MALLOC_GRANS. */ |
| 279 |
q = (ptr_t)AO_load((volatile AO_t *)&p->_freelists[i][j]); |
| 280 |
if ((word)q > HBLKSIZE) |
| 281 |
GC_set_fl_marks(q); |
| 282 |
} |
| 283 |
# ifdef GC_GCJ_SUPPORT |
| 284 |
if (EXPECT(j > 0, TRUE)) { |
| 285 |
q = (ptr_t)AO_load((volatile AO_t *)&p->gcj_freelists[j]); |
| 286 |
if ((word)q > HBLKSIZE) |
| 287 |
GC_set_fl_marks(q); |
| 288 |
} |
| 289 |
# endif |
| 290 |
} |
| 291 |
} |
| 292 |
|
| 293 |
#if defined(GC_ASSERTIONS) |
| 294 |
/* Check that all thread-local free-lists in p are completely marked. */ |
| 295 |
void GC_check_tls_for(GC_tlfs p) |
| 296 |
{ |
| 297 |
int i, j; |
| 298 |
|
| 299 |
for (j = 1; j < TINY_FREELISTS; ++j) { |
| 300 |
for (i = 0; i < THREAD_FREELISTS_KINDS; ++i) { |
| 301 |
GC_check_fl_marks(&p->_freelists[i][j]); |
| 302 |
} |
| 303 |
# ifdef GC_GCJ_SUPPORT |
| 304 |
GC_check_fl_marks(&p->gcj_freelists[j]); |
| 305 |
# endif |
| 306 |
} |
| 307 |
} |
| 308 |
#endif /* GC_ASSERTIONS */ |
| 309 |
|
| 310 |
#endif /* THREAD_LOCAL_ALLOC */ |