Debian Codegraph @

src/backend/storage/lmgr/lwlock.c

raw
1 /*-------------------------------------------------------------------------
2 *
3 * lwlock.c
4 * Lightweight lock manager
5 *
6 * Lightweight locks are intended primarily to provide mutual exclusion of
7 * access to shared-memory data structures. Therefore, they offer both
8 * exclusive and shared lock modes (to support read/write and read-only
9 * access to a shared object). There are few other frammishes. User-level
10 * locking should be done with the full lock manager --- which depends on
11 * LWLocks to protect its shared state.
12 *
13 * In addition to exclusive and shared modes, lightweight locks can be used to
14 * wait until a variable changes value. The variable is initially not set
15 * when the lock is acquired with LWLockAcquire, i.e. it remains set to the
16 * value it was set to when the lock was released last, and can be updated
17 * without releasing the lock by calling LWLockUpdateVar. LWLockWaitForVar
18 * waits for the variable to be updated, or until the lock is free. When
19 * releasing the lock with LWLockReleaseClearVar() the value can be set to an
20 * appropriate value for a free lock. The meaning of the variable is up to
21 * the caller, the lightweight lock code just assigns and compares it.
22 *
23 * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
24 * Portions Copyright (c) 1994, Regents of the University of California
25 *
26 * IDENTIFICATION
27 * src/backend/storage/lmgr/lwlock.c
28 *
29 * NOTES:
30 *
31 * This used to be a pretty straight forward reader-writer lock
32 * implementation, in which the internal state was protected by a
33 * spinlock. Unfortunately the overhead of taking the spinlock proved to be
34 * too high for workloads/locks that were taken in shared mode very
35 * frequently. Often we were spinning in the (obviously exclusive) spinlock,
36 * while trying to acquire a shared lock that was actually free.
37 *
38 * Thus a new implementation was devised that provides wait-free shared lock
39 * acquisition for locks that aren't exclusively locked.
40 *
41 * The basic idea is to have a single atomic variable 'lockcount' instead of
42 * the formerly separate shared and exclusive counters and to use atomic
43 * operations to acquire the lock. That's fairly easy to do for plain
44 * rw-spinlocks, but a lot harder for something like LWLocks that want to wait
45 * in the OS.
46 *
47 * For lock acquisition we use an atomic compare-and-exchange on the lockcount
48 * variable. For exclusive lock we swap in a sentinel value
49 * (LW_VAL_EXCLUSIVE), for shared locks we count the number of holders.
50 *
51 * To release the lock we use an atomic decrement to release the lock. If the
52 * new value is zero (we get that atomically), we know we can/have to release
53 * waiters.
54 *
55 * Obviously it is important that the sentinel value for exclusive locks
56 * doesn't conflict with the maximum number of possible share lockers -
57 * luckily MAX_BACKENDS makes that easily possible.
58 *
59 *
60 * The attentive reader might have noticed that naively doing the above has a
61 * glaring race condition: We try to lock using the atomic operations and
62 * notice that we have to wait. Unfortunately by the time we have finished
63 * queuing, the former locker very well might have already finished its
64 * work. That's problematic because we're now stuck waiting inside the OS.
65
66 * To mitigate those races we use a two phased attempt at locking:
67 * Phase 1: Try to do it atomically, if we succeed, nice
68 * Phase 2: Add ourselves to the waitqueue of the lock
69 * Phase 3: Try to grab the lock again, if we succeed, remove ourselves from
70 * the queue
71 * Phase 4: Sleep till wake-up, goto Phase 1
72 *
73 * This protects us against the problem from above as nobody can release too
74 * quick, before we're queued, since after Phase 2 we're already queued.
75 * -------------------------------------------------------------------------
76 */
77 #include "postgres.h"
78
79 #include "miscadmin.h"
80 #include "pg_trace.h"
81 #include "pgstat.h"
82 #include "port/pg_bitutils.h"
83 #include "storage/proc.h"
84 #include "storage/proclist.h"
85 #include "storage/procnumber.h"
86 #include "storage/spin.h"
87 #include "utils/memutils.h"
88
89 #ifdef LWLOCK_STATS
90 #include "utils/hsearch.h"
91 #endif
92
93
94 #define LW_FLAG_HAS_WAITERS ((uint32) 1 << 31)
95 #define LW_FLAG_RELEASE_OK ((uint32) 1 << 30)
96 #define LW_FLAG_LOCKED ((uint32) 1 << 29)
97 #define LW_FLAG_BITS 3
98 #define LW_FLAG_MASK (((1<<LW_FLAG_BITS)-1)<<(32-LW_FLAG_BITS))
99
100 /* assumes MAX_BACKENDS is a (power of 2) - 1, checked below */
101 #define LW_VAL_EXCLUSIVE (MAX_BACKENDS + 1)
102 #define LW_VAL_SHARED 1
103
104 /* already (power of 2)-1, i.e. suitable for a mask */
105 #define LW_SHARED_MASK MAX_BACKENDS
106 #define LW_LOCK_MASK (MAX_BACKENDS | LW_VAL_EXCLUSIVE)
107
108
109 StaticAssertDecl(((MAX_BACKENDS + 1) & MAX_BACKENDS) == 0,
110 "MAX_BACKENDS + 1 needs to be a power of 2");
111
112 StaticAssertDecl((MAX_BACKENDS & LW_FLAG_MASK) == 0,
113 "MAX_BACKENDS and LW_FLAG_MASK overlap");
114
115 StaticAssertDecl((LW_VAL_EXCLUSIVE & LW_FLAG_MASK) == 0,
116 "LW_VAL_EXCLUSIVE and LW_FLAG_MASK overlap");
117
118 /*
119 * There are three sorts of LWLock "tranches":
120 *
121 * 1. The individually-named locks defined in lwlocklist.h each have their
122 * own tranche. We absorb the names of these tranches from there into
123 * BuiltinTrancheNames here.
124 *
125 * 2. There are some predefined tranches for built-in groups of locks.
126 * These are listed in enum BuiltinTrancheIds in lwlock.h, and their names
127 * appear in BuiltinTrancheNames[] below.
128 *
129 * 3. Extensions can create new tranches, via either RequestNamedLWLockTranche
130 * or LWLockRegisterTranche. The names of these that are known in the current
131 * process appear in LWLockTrancheNames[].
132 *
133 * All these names are user-visible as wait event names, so choose with care
134 * ... and do not forget to update the documentation's list of wait events.
135 */
136 static const char *const BuiltinTrancheNames[] = {
137 #define PG_LWLOCK(id, lockname) [id] = CppAsString(lockname),
138 #include "storage/lwlocklist.h"
139 #undef PG_LWLOCK
140 [LWTRANCHE_XACT_BUFFER] = "XactBuffer",
141 [LWTRANCHE_COMMITTS_BUFFER] = "CommitTsBuffer",
142 [LWTRANCHE_SUBTRANS_BUFFER] = "SubtransBuffer",
143 [LWTRANCHE_MULTIXACTOFFSET_BUFFER] = "MultiXactOffsetBuffer",
144 [LWTRANCHE_MULTIXACTMEMBER_BUFFER] = "MultiXactMemberBuffer",
145 [LWTRANCHE_NOTIFY_BUFFER] = "NotifyBuffer",
146 [LWTRANCHE_SERIAL_BUFFER] = "SerialBuffer",
147 [LWTRANCHE_WAL_INSERT] = "WALInsert",
148 [LWTRANCHE_BUFFER_CONTENT] = "BufferContent",
149 [LWTRANCHE_REPLICATION_ORIGIN_STATE] = "ReplicationOriginState",
150 [LWTRANCHE_REPLICATION_SLOT_IO] = "ReplicationSlotIO",
151 [LWTRANCHE_LOCK_FASTPATH] = "LockFastPath",
152 [LWTRANCHE_BUFFER_MAPPING] = "BufferMapping",
153 [LWTRANCHE_LOCK_MANAGER] = "LockManager",
154 [LWTRANCHE_PREDICATE_LOCK_MANAGER] = "PredicateLockManager",
155 [LWTRANCHE_PARALLEL_HASH_JOIN] = "ParallelHashJoin",
156 [LWTRANCHE_PARALLEL_BTREE_SCAN] = "ParallelBtreeScan",
157 [LWTRANCHE_PARALLEL_QUERY_DSA] = "ParallelQueryDSA",
158 [LWTRANCHE_PER_SESSION_DSA] = "PerSessionDSA",
159 [LWTRANCHE_PER_SESSION_RECORD_TYPE] = "PerSessionRecordType",
160 [LWTRANCHE_PER_SESSION_RECORD_TYPMOD] = "PerSessionRecordTypmod",
161 [LWTRANCHE_SHARED_TUPLESTORE] = "SharedTupleStore",
162 [LWTRANCHE_SHARED_TIDBITMAP] = "SharedTidBitmap",
163 [LWTRANCHE_PARALLEL_APPEND] = "ParallelAppend",
164 [LWTRANCHE_PER_XACT_PREDICATE_LIST] = "PerXactPredicateList",
165 [LWTRANCHE_PGSTATS_DSA] = "PgStatsDSA",
166 [LWTRANCHE_PGSTATS_HASH] = "PgStatsHash",
167 [LWTRANCHE_PGSTATS_DATA] = "PgStatsData",
168 [LWTRANCHE_LAUNCHER_DSA] = "LogicalRepLauncherDSA",
169 [LWTRANCHE_LAUNCHER_HASH] = "LogicalRepLauncherHash",
170 [LWTRANCHE_DSM_REGISTRY_DSA] = "DSMRegistryDSA",
171 [LWTRANCHE_DSM_REGISTRY_HASH] = "DSMRegistryHash",
172 [LWTRANCHE_COMMITTS_SLRU] = "CommitTsSLRU",
173 [LWTRANCHE_MULTIXACTOFFSET_SLRU] = "MultiXactOffsetSLRU",
174 [LWTRANCHE_MULTIXACTMEMBER_SLRU] = "MultiXactMemberSLRU",
175 [LWTRANCHE_NOTIFY_SLRU] = "NotifySLRU",
176 [LWTRANCHE_SERIAL_SLRU] = "SerialSLRU",
177 [LWTRANCHE_SUBTRANS_SLRU] = "SubtransSLRU",
178 [LWTRANCHE_XACT_SLRU] = "XactSLRU",
179 [LWTRANCHE_PARALLEL_VACUUM_DSA] = "ParallelVacuumDSA",
180 [LWTRANCHE_AIO_URING_COMPLETION] = "AioUringCompletion",
181 };
182
183 StaticAssertDecl(lengthof(BuiltinTrancheNames) ==
184 LWTRANCHE_FIRST_USER_DEFINED,
185 "missing entries in BuiltinTrancheNames[]");
186
187 /*
188 * This is indexed by tranche ID minus LWTRANCHE_FIRST_USER_DEFINED, and
189 * stores the names of all dynamically-created tranches known to the current
190 * process. Any unused entries in the array will contain NULL.
191 */
192 static const char **LWLockTrancheNames = NULL;
193 static int LWLockTrancheNamesAllocated = 0;
194
195 /*
196 * This points to the main array of LWLocks in shared memory. Backends inherit
197 * the pointer by fork from the postmaster (except in the EXEC_BACKEND case,
198 * where we have special measures to pass it down).
199 */
200 LWLockPadded *MainLWLockArray = NULL;
201
202 /*
203 * We use this structure to keep track of locked LWLocks for release
204 * during error recovery. Normally, only a few will be held at once, but
205 * occasionally the number can be much higher; for example, the pg_buffercache
206 * extension locks all buffer partitions simultaneously.
207 */
208 #define MAX_SIMUL_LWLOCKS 200
209
210 /* struct representing the LWLocks we're holding */
211 typedef struct LWLockHandle
212 {
213 LWLock *lock;
214 LWLockMode mode;
215 } LWLockHandle;
216
217 static int num_held_lwlocks = 0;
218 static LWLockHandle held_lwlocks[MAX_SIMUL_LWLOCKS];
219
220 /* struct representing the LWLock tranche request for named tranche */
221 typedef struct NamedLWLockTrancheRequest
222 {
223 char tranche_name[NAMEDATALEN];
224 int num_lwlocks;
225 } NamedLWLockTrancheRequest;
226
227 static NamedLWLockTrancheRequest *NamedLWLockTrancheRequestArray = NULL;
228 static int NamedLWLockTrancheRequestsAllocated = 0;
229
230 /*
231 * NamedLWLockTrancheRequests is both the valid length of the request array,
232 * and the length of the shared-memory NamedLWLockTrancheArray later on.
233 * This variable and NamedLWLockTrancheArray are non-static so that
234 * postmaster.c can copy them to child processes in EXEC_BACKEND builds.
235 */
236 int NamedLWLockTrancheRequests = 0;
237
238 /* points to data in shared memory: */
239 NamedLWLockTranche *NamedLWLockTrancheArray = NULL;
240
241 static void InitializeLWLocks(void);
242 static inline void LWLockReportWaitStart(LWLock *lock);
243 static inline void LWLockReportWaitEnd(void);
244 static const char *GetLWTrancheName(uint16 trancheId);
245
246 #define T_NAME(lock) \
247 GetLWTrancheName((lock)->tranche)
248
249 #ifdef LWLOCK_STATS
250 typedef struct lwlock_stats_key
251 {
252 int tranche;
253 void *instance;
254 } lwlock_stats_key;
255
256 typedef struct lwlock_stats
257 {
258 lwlock_stats_key key;
259 int sh_acquire_count;
260 int ex_acquire_count;
261 int block_count;
262 int dequeue_self_count;
263 int spin_delay_count;
264 } lwlock_stats;
265
266 static HTAB *lwlock_stats_htab;
267 static lwlock_stats lwlock_stats_dummy;
268 #endif
269
270 #ifdef LOCK_DEBUG
271 bool Trace_lwlocks = false;
272
273 inline static void
274 PRINT_LWDEBUG(const char *where, LWLock *lock, LWLockMode mode)
275 {
276 /* hide statement & context here, otherwise the log is just too verbose */
277 if (Trace_lwlocks)
278 {
279 uint32 state = pg_atomic_read_u32(&lock->state);
280
281 ereport(LOG,
282 (errhidestmt(true),
283 errhidecontext(true),
284 errmsg_internal("%d: %s(%s %p): excl %u shared %u haswaiters %u waiters %u rOK %d",
285 MyProcPid,
286 where, T_NAME(lock), lock,
287 (state & LW_VAL_EXCLUSIVE) != 0,
288 state & LW_SHARED_MASK,
289 (state & LW_FLAG_HAS_WAITERS) != 0,
290 pg_atomic_read_u32(&lock->nwaiters),
291 (state & LW_FLAG_RELEASE_OK) != 0)));
292 }
293 }
294
295 inline static void
296 LOG_LWDEBUG(const char *where, LWLock *lock, const char *msg)
297 {
298 /* hide statement & context here, otherwise the log is just too verbose */
299 if (Trace_lwlocks)
300 {
301 ereport(LOG,
302 (errhidestmt(true),
303 errhidecontext(true),
304 errmsg_internal("%s(%s %p): %s", where,
305 T_NAME(lock), lock, msg)));
306 }
307 }
308
309 #else /* not LOCK_DEBUG */
310 #define PRINT_LWDEBUG(a,b,c) ((void)0)
311 #define LOG_LWDEBUG(a,b,c) ((void)0)
312 #endif /* LOCK_DEBUG */
313
314 #ifdef LWLOCK_STATS
315
316 static void init_lwlock_stats(void);
317 static void print_lwlock_stats(int code, Datum arg);
318 static lwlock_stats * get_lwlock_stats_entry(LWLock *lock);
319
320 static void
321 init_lwlock_stats(void)
322 {
323 HASHCTL ctl;
324 static MemoryContext lwlock_stats_cxt = NULL;
325 static bool exit_registered = false;
326
327 if (lwlock_stats_cxt != NULL)
328 MemoryContextDelete(lwlock_stats_cxt);
329
330 /*
331 * The LWLock stats will be updated within a critical section, which
332 * requires allocating new hash entries. Allocations within a critical
333 * section are normally not allowed because running out of memory would
334 * lead to a PANIC, but LWLOCK_STATS is debugging code that's not normally
335 * turned on in production, so that's an acceptable risk. The hash entries
336 * are small, so the risk of running out of memory is minimal in practice.
337 */
338 lwlock_stats_cxt = AllocSetContextCreate(TopMemoryContext,
339 "LWLock stats",
340 ALLOCSET_DEFAULT_SIZES);
341 MemoryContextAllowInCriticalSection(lwlock_stats_cxt, true);
342
343 ctl.keysize = sizeof(lwlock_stats_key);
344 ctl.entrysize = sizeof(lwlock_stats);
345 ctl.hcxt = lwlock_stats_cxt;
346 lwlock_stats_htab = hash_create("lwlock stats", 16384, &ctl,
347 HASH_ELEM | HASH_BLOBS | HASH_CONTEXT);
348 if (!exit_registered)
349 {
350 on_shmem_exit(print_lwlock_stats, 0);
351 exit_registered = true;
352 }
353 }
354
355 static void
356 print_lwlock_stats(int code, Datum arg)
357 {
358 HASH_SEQ_STATUS scan;
359 lwlock_stats *lwstats;
360
361 hash_seq_init(&scan, lwlock_stats_htab);
362
363 /* Grab an LWLock to keep different backends from mixing reports */
364 LWLockAcquire(&MainLWLockArray[0].lock, LW_EXCLUSIVE);
365
366 while ((lwstats = (lwlock_stats *) hash_seq_search(&scan)) != NULL)
367 {
368 fprintf(stderr,
369 "PID %d lwlock %s %p: shacq %u exacq %u blk %u spindelay %u dequeue self %u\n",
370 MyProcPid, GetLWTrancheName(lwstats->key.tranche),
371 lwstats->key.instance, lwstats->sh_acquire_count,
372 lwstats->ex_acquire_count, lwstats->block_count,
373 lwstats->spin_delay_count, lwstats->dequeue_self_count);
374 }
375
376 LWLockRelease(&MainLWLockArray[0].lock);
377 }
378
379 static lwlock_stats *
380 get_lwlock_stats_entry(LWLock *lock)
381 {
382 lwlock_stats_key key;
383 lwlock_stats *lwstats;
384 bool found;
385
386 /*
387 * During shared memory initialization, the hash table doesn't exist yet.
388 * Stats of that phase aren't very interesting, so just collect operations
389 * on all locks in a single dummy entry.
390 */
391 if (lwlock_stats_htab == NULL)
392 return &lwlock_stats_dummy;
393
394 /* Fetch or create the entry. */
395 MemSet(&key, 0, sizeof(key));
396 key.tranche = lock->tranche;
397 key.instance = lock;
398 lwstats = hash_search(lwlock_stats_htab, &key, HASH_ENTER, &found);
399 if (!found)
400 {
401 lwstats->sh_acquire_count = 0;
402 lwstats->ex_acquire_count = 0;
403 lwstats->block_count = 0;
404 lwstats->dequeue_self_count = 0;
405 lwstats->spin_delay_count = 0;
406 }
407 return lwstats;
408 }
409 #endif /* LWLOCK_STATS */
410
411
412 /*
413 * Compute number of LWLocks required by named tranches. These will be
414 * allocated in the main array.
415 */
416 static int
417 NumLWLocksForNamedTranches(void)
418 {
419 int numLocks = 0;
420 int i;
421
422 for (i = 0; i < NamedLWLockTrancheRequests; i++)
423 numLocks += NamedLWLockTrancheRequestArray[i].num_lwlocks;
424
425 return numLocks;
426 }
427
428 /*
429 * Compute shmem space needed for LWLocks and named tranches.
430 */
431 Size
432 LWLockShmemSize(void)
433 {
434 Size size;
435 int i;
436 int numLocks = NUM_FIXED_LWLOCKS;
437
438 /* Calculate total number of locks needed in the main array. */
439 numLocks += NumLWLocksForNamedTranches();
440
441 /* Space for the LWLock array. */
442 size = mul_size(numLocks, sizeof(LWLockPadded));
443
444 /* Space for dynamic allocation counter, plus room for alignment. */
445 size = add_size(size, sizeof(int) + LWLOCK_PADDED_SIZE);
446
447 /* space for named tranches. */
448 size = add_size(size, mul_size(NamedLWLockTrancheRequests, sizeof(NamedLWLockTranche)));
449
450 /* space for name of each tranche. */
451 for (i = 0; i < NamedLWLockTrancheRequests; i++)
452 size = add_size(size, strlen(NamedLWLockTrancheRequestArray[i].tranche_name) + 1);
453
454 return size;
455 }
456
457 /*
458 * Allocate shmem space for the main LWLock array and all tranches and
459 * initialize it. We also register extension LWLock tranches here.
460 */
461 void
462 CreateLWLocks(void)
463 {
464 if (!IsUnderPostmaster)
465 {
466 Size spaceLocks = LWLockShmemSize();
467 int *LWLockCounter;
468 char *ptr;
469
470 /* Allocate space */
471 ptr = (char *) ShmemAlloc(spaceLocks);
472
473 /* Leave room for dynamic allocation of tranches */
474 ptr += sizeof(int);
475
476 /* Ensure desired alignment of LWLock array */
477 ptr += LWLOCK_PADDED_SIZE - ((uintptr_t) ptr) % LWLOCK_PADDED_SIZE;
478
479 MainLWLockArray = (LWLockPadded *) ptr;
480
481 /*
482 * Initialize the dynamic-allocation counter for tranches, which is
483 * stored just before the first LWLock.
484 */
485 LWLockCounter = (int *) ((char *) MainLWLockArray - sizeof(int));
486 *LWLockCounter = LWTRANCHE_FIRST_USER_DEFINED;
487
488 /* Initialize all LWLocks */
489 InitializeLWLocks();
490 }
491
492 /* Register named extension LWLock tranches in the current process. */
493 for (int i = 0; i < NamedLWLockTrancheRequests; i++)
494 LWLockRegisterTranche(NamedLWLockTrancheArray[i].trancheId,
495 NamedLWLockTrancheArray[i].trancheName);
496 }
497
498 /*
499 * Initialize LWLocks that are fixed and those belonging to named tranches.
500 */
501 static void
502 InitializeLWLocks(void)
503 {
504 int numNamedLocks = NumLWLocksForNamedTranches();
505 int id;
506 int i;
507 int j;
508 LWLockPadded *lock;
509
510 /* Initialize all individual LWLocks in main array */
511 for (id = 0, lock = MainLWLockArray; id < NUM_INDIVIDUAL_LWLOCKS; id++, lock++)
512 LWLockInitialize(&lock->lock, id);
513
514 /* Initialize buffer mapping LWLocks in main array */
515 lock = MainLWLockArray + BUFFER_MAPPING_LWLOCK_OFFSET;
516 for (id = 0; id < NUM_BUFFER_PARTITIONS; id++, lock++)
517 LWLockInitialize(&lock->lock, LWTRANCHE_BUFFER_MAPPING);
518
519 /* Initialize lmgrs' LWLocks in main array */
520 lock = MainLWLockArray + LOCK_MANAGER_LWLOCK_OFFSET;
521 for (id = 0; id < NUM_LOCK_PARTITIONS; id++, lock++)
522 LWLockInitialize(&lock->lock, LWTRANCHE_LOCK_MANAGER);
523
524 /* Initialize predicate lmgrs' LWLocks in main array */
525 lock = MainLWLockArray + PREDICATELOCK_MANAGER_LWLOCK_OFFSET;
526 for (id = 0; id < NUM_PREDICATELOCK_PARTITIONS; id++, lock++)
527 LWLockInitialize(&lock->lock, LWTRANCHE_PREDICATE_LOCK_MANAGER);
528
529 /*
530 * Copy the info about any named tranches into shared memory (so that
531 * other processes can see it), and initialize the requested LWLocks.
532 */
533 if (NamedLWLockTrancheRequests > 0)
534 {
535 char *trancheNames;
536
537 NamedLWLockTrancheArray = (NamedLWLockTranche *)
538 &MainLWLockArray[NUM_FIXED_LWLOCKS + numNamedLocks];
539
540 trancheNames = (char *) NamedLWLockTrancheArray +
541 (NamedLWLockTrancheRequests * sizeof(NamedLWLockTranche));
542 lock = &MainLWLockArray[NUM_FIXED_LWLOCKS];
543
544 for (i = 0; i < NamedLWLockTrancheRequests; i++)
545 {
546 NamedLWLockTrancheRequest *request;
547 NamedLWLockTranche *tranche;
548 char *name;
549
550 request = &NamedLWLockTrancheRequestArray[i];
551 tranche = &NamedLWLockTrancheArray[i];
552
553 name = trancheNames;
554 trancheNames += strlen(request->tranche_name) + 1;
555 strcpy(name, request->tranche_name);
556 tranche->trancheId = LWLockNewTrancheId();
557 tranche->trancheName = name;
558
559 for (j = 0; j < request->num_lwlocks; j++, lock++)
560 LWLockInitialize(&lock->lock, tranche->trancheId);
561 }
562 }
563 }
564
565 /*
566 * InitLWLockAccess - initialize backend-local state needed to hold LWLocks
567 */
568 void
569 InitLWLockAccess(void)
570 {
571 #ifdef LWLOCK_STATS
572 init_lwlock_stats();
573 #endif
574 }
575
576 /*
577 * GetNamedLWLockTranche - returns the base address of LWLock from the
578 * specified tranche.
579 *
580 * Caller needs to retrieve the requested number of LWLocks starting from
581 * the base lock address returned by this API. This can be used for
582 * tranches that are requested by using RequestNamedLWLockTranche() API.
583 */
584 LWLockPadded *
585 GetNamedLWLockTranche(const char *tranche_name)
586 {
587 int lock_pos;
588 int i;
589
590 /*
591 * Obtain the position of base address of LWLock belonging to requested
592 * tranche_name in MainLWLockArray. LWLocks for named tranches are placed
593 * in MainLWLockArray after fixed locks.
594 */
595 lock_pos = NUM_FIXED_LWLOCKS;
596 for (i = 0; i < NamedLWLockTrancheRequests; i++)
597 {
598 if (strcmp(NamedLWLockTrancheRequestArray[i].tranche_name,
599 tranche_name) == 0)
600 return &MainLWLockArray[lock_pos];
601
602 lock_pos += NamedLWLockTrancheRequestArray[i].num_lwlocks;
603 }
604
605 elog(ERROR, "requested tranche is not registered");
606
607 /* just to keep compiler quiet */
608 return NULL;
609 }
610
611 /*
612 * Allocate a new tranche ID.
613 */
614 int
615 LWLockNewTrancheId(void)
616 {
617 int result;
618 int *LWLockCounter;
619
620 LWLockCounter = (int *) ((char *) MainLWLockArray - sizeof(int));
621 /* We use the ShmemLock spinlock to protect LWLockCounter */
622 SpinLockAcquire(ShmemLock);
623 result = (*LWLockCounter)++;
624 SpinLockRelease(ShmemLock);
625
626 return result;
627 }
628
629 /*
630 * Register a dynamic tranche name in the lookup table of the current process.
631 *
632 * This routine will save a pointer to the tranche name passed as an argument,
633 * so the name should be allocated in a backend-lifetime context
634 * (shared memory, TopMemoryContext, static constant, or similar).
635 *
636 * The tranche name will be user-visible as a wait event name, so try to
637 * use a name that fits the style for those.
638 */
639 void
640 LWLockRegisterTranche(int tranche_id, const char *tranche_name)
641 {
642 /* This should only be called for user-defined tranches. */
643 if (tranche_id < LWTRANCHE_FIRST_USER_DEFINED)
644 return;
645
646 /* Convert to array index. */
647 tranche_id -= LWTRANCHE_FIRST_USER_DEFINED;
648
649 /* If necessary, create or enlarge array. */
650 if (tranche_id >= LWLockTrancheNamesAllocated)
651 {
652 int newalloc;
653
654 newalloc = pg_nextpower2_32(Max(8, tranche_id + 1));
655
656 if (LWLockTrancheNames == NULL)
657 LWLockTrancheNames = (const char **)
658 MemoryContextAllocZero(TopMemoryContext,
659 newalloc * sizeof(char *));
660 else
661 LWLockTrancheNames =
662 repalloc0_array(LWLockTrancheNames, const char *, LWLockTrancheNamesAllocated, newalloc);
663 LWLockTrancheNamesAllocated = newalloc;
664 }
665
666 LWLockTrancheNames[tranche_id] = tranche_name;
667 }
668
669 /*
670 * RequestNamedLWLockTranche
671 * Request that extra LWLocks be allocated during postmaster
672 * startup.
673 *
674 * This may only be called via the shmem_request_hook of a library that is
675 * loaded into the postmaster via shared_preload_libraries. Calls from
676 * elsewhere will fail.
677 *
678 * The tranche name will be user-visible as a wait event name, so try to
679 * use a name that fits the style for those.
680 */
681 void
682 RequestNamedLWLockTranche(const char *tranche_name, int num_lwlocks)
683 {
684 NamedLWLockTrancheRequest *request;
685
686 if (!process_shmem_requests_in_progress)
687 elog(FATAL, "cannot request additional LWLocks outside shmem_request_hook");
688
689 if (NamedLWLockTrancheRequestArray == NULL)
690 {
691 NamedLWLockTrancheRequestsAllocated = 16;
692 NamedLWLockTrancheRequestArray = (NamedLWLockTrancheRequest *)
693 MemoryContextAlloc(TopMemoryContext,
694 NamedLWLockTrancheRequestsAllocated
695 * sizeof(NamedLWLockTrancheRequest));
696 }
697
698 if (NamedLWLockTrancheRequests >= NamedLWLockTrancheRequestsAllocated)
699 {
700 int i = pg_nextpower2_32(NamedLWLockTrancheRequests + 1);
701
702 NamedLWLockTrancheRequestArray = (NamedLWLockTrancheRequest *)
703 repalloc(NamedLWLockTrancheRequestArray,
704 i * sizeof(NamedLWLockTrancheRequest));
705 NamedLWLockTrancheRequestsAllocated = i;
706 }
707
708 request = &NamedLWLockTrancheRequestArray[NamedLWLockTrancheRequests];
709 Assert(strlen(tranche_name) + 1 <= NAMEDATALEN);
710 strlcpy(request->tranche_name, tranche_name, NAMEDATALEN);
711 request->num_lwlocks = num_lwlocks;
712 NamedLWLockTrancheRequests++;
713 }
714
715 /*
716 * LWLockInitialize - initialize a new lwlock; it's initially unlocked
717 */
718 void
719 LWLockInitialize(LWLock *lock, int tranche_id)
720 {
721 pg_atomic_init_u32(&lock->state, LW_FLAG_RELEASE_OK);
722 #ifdef LOCK_DEBUG
723 pg_atomic_init_u32(&lock->nwaiters, 0);
724 #endif
725 lock->tranche = tranche_id;
726 proclist_init(&lock->waiters);
727 }
728
729 /*
730 * Report start of wait event for light-weight locks.
731 *
732 * This function will be used by all the light-weight lock calls which
733 * needs to wait to acquire the lock. This function distinguishes wait
734 * event based on tranche and lock id.
735 */
736 static inline void
737 LWLockReportWaitStart(LWLock *lock)
738 {
739 pgstat_report_wait_start(PG_WAIT_LWLOCK | lock->tranche);
740 }
741
742 /*
743 * Report end of wait event for light-weight locks.
744 */
745 static inline void
746 LWLockReportWaitEnd(void)
747 {
748 pgstat_report_wait_end();
749 }
750
751 /*
752 * Return the name of an LWLock tranche.
753 */
754 static const char *
755 GetLWTrancheName(uint16 trancheId)
756 {
757 /* Built-in tranche or individual LWLock? */
758 if (trancheId < LWTRANCHE_FIRST_USER_DEFINED)
759 return BuiltinTrancheNames[trancheId];
760
761 /*
762 * It's an extension tranche, so look in LWLockTrancheNames[]. However,
763 * it's possible that the tranche has never been registered in the current
764 * process, in which case give up and return "extension".
765 */
766 trancheId -= LWTRANCHE_FIRST_USER_DEFINED;
767
768 if (trancheId >= LWLockTrancheNamesAllocated ||
769 LWLockTrancheNames[trancheId] == NULL)
770 return "extension";
771
772 return LWLockTrancheNames[trancheId];
773 }
774
775 /*
776 * Return an identifier for an LWLock based on the wait class and event.
777 */
778 const char *
779 GetLWLockIdentifier(uint32 classId, uint16 eventId)
780 {
781 Assert(classId == PG_WAIT_LWLOCK);
782 /* The event IDs are just tranche numbers. */
783 return GetLWTrancheName(eventId);
784 }
785
786 /*
787 * Internal function that tries to atomically acquire the lwlock in the passed
788 * in mode.
789 *
790 * This function will not block waiting for a lock to become free - that's the
791 * caller's job.
792 *
793 * Returns true if the lock isn't free and we need to wait.
794 */
795 static bool
796 LWLockAttemptLock(LWLock *lock, LWLockMode mode)
797 {
798 uint32 old_state;
799
800 Assert(mode == LW_EXCLUSIVE || mode == LW_SHARED);
801
802 /*
803 * Read once outside the loop, later iterations will get the newer value
804 * via compare & exchange.
805 */
806 old_state = pg_atomic_read_u32(&lock->state);
807
808 /* loop until we've determined whether we could acquire the lock or not */
809 while (true)
810 {
811 uint32 desired_state;
812 bool lock_free;
813
814 desired_state = old_state;
815
816 if (mode == LW_EXCLUSIVE)
817 {
818 lock_free = (old_state & LW_LOCK_MASK) == 0;
819 if (lock_free)
820 desired_state += LW_VAL_EXCLUSIVE;
821 }
822 else
823 {
824 lock_free = (old_state & LW_VAL_EXCLUSIVE) == 0;
825 if (lock_free)
826 desired_state += LW_VAL_SHARED;
827 }
828
829 /*
830 * Attempt to swap in the state we are expecting. If we didn't see
831 * lock to be free, that's just the old value. If we saw it as free,
832 * we'll attempt to mark it acquired. The reason that we always swap
833 * in the value is that this doubles as a memory barrier. We could try
834 * to be smarter and only swap in values if we saw the lock as free,
835 * but benchmark haven't shown it as beneficial so far.
836 *
837 * Retry if the value changed since we last looked at it.
838 */
839 if (pg_atomic_compare_exchange_u32(&lock->state,
840 &old_state, desired_state))
841 {
842 if (lock_free)
843 {
844 /* Great! Got the lock. */
845 #ifdef LOCK_DEBUG
846 if (mode == LW_EXCLUSIVE)
847 lock->owner = MyProc;
848 #endif
849 return false;
850 }
851 else
852 return true; /* somebody else has the lock */
853 }
854 }
855 pg_unreachable();
856 }
857
858 /*
859 * Lock the LWLock's wait list against concurrent activity.
860 *
861 * NB: even though the wait list is locked, non-conflicting lock operations
862 * may still happen concurrently.
863 *
864 * Time spent holding mutex should be short!
865 */
866 static void
867 LWLockWaitListLock(LWLock *lock)
868 {
869 uint32 old_state;
870 #ifdef LWLOCK_STATS
871 lwlock_stats *lwstats;
872 uint32 delays = 0;
873
874 lwstats = get_lwlock_stats_entry(lock);
875 #endif
876
877 while (true)
878 {
879 /* always try once to acquire lock directly */
880 old_state = pg_atomic_fetch_or_u32(&lock->state, LW_FLAG_LOCKED);
881 if (!(old_state & LW_FLAG_LOCKED))
882 break; /* got lock */
883
884 /* and then spin without atomic operations until lock is released */
885 {
886 SpinDelayStatus delayStatus;
887
888 init_local_spin_delay(&delayStatus);
889
890 while (old_state & LW_FLAG_LOCKED)
891 {
892 perform_spin_delay(&delayStatus);
893 old_state = pg_atomic_read_u32(&lock->state);
894 }
895 #ifdef LWLOCK_STATS
896 delays += delayStatus.delays;
897 #endif
898 finish_spin_delay(&delayStatus);
899 }
900
901 /*
902 * Retry. The lock might obviously already be re-acquired by the time
903 * we're attempting to get it again.
904 */
905 }
906
907 #ifdef LWLOCK_STATS
908 lwstats->spin_delay_count += delays;
909 #endif
910 }
911
912 /*
913 * Unlock the LWLock's wait list.
914 *
915 * Note that it can be more efficient to manipulate flags and release the
916 * locks in a single atomic operation.
917 */
918 static void
919 LWLockWaitListUnlock(LWLock *lock)
920 {
921 uint32 old_state PG_USED_FOR_ASSERTS_ONLY;
922
923 old_state = pg_atomic_fetch_and_u32(&lock->state, ~LW_FLAG_LOCKED);
924
925 Assert(old_state & LW_FLAG_LOCKED);
926 }
927
928 /*
929 * Wakeup all the lockers that currently have a chance to acquire the lock.
930 */
931 static void
932 LWLockWakeup(LWLock *lock)
933 {
934 bool new_release_ok;
935 bool wokeup_somebody = false;
936 proclist_head wakeup;
937 proclist_mutable_iter iter;
938
939 proclist_init(&wakeup);
940
941 new_release_ok = true;
942
943 /* lock wait list while collecting backends to wake up */
944 LWLockWaitListLock(lock);
945
946 proclist_foreach_modify(iter, &lock->waiters, lwWaitLink)
947 {
948 PGPROC *waiter = GetPGProcByNumber(iter.cur);
949
950 if (wokeup_somebody && waiter->lwWaitMode == LW_EXCLUSIVE)
951 continue;
952
953 proclist_delete(&lock->waiters, iter.cur, lwWaitLink);
954 proclist_push_tail(&wakeup, iter.cur, lwWaitLink);
955
956 if (waiter->lwWaitMode != LW_WAIT_UNTIL_FREE)
957 {
958 /*
959 * Prevent additional wakeups until retryer gets to run. Backends
960 * that are just waiting for the lock to become free don't retry
961 * automatically.
962 */
963 new_release_ok = false;
964
965 /*
966 * Don't wakeup (further) exclusive locks.
967 */
968 wokeup_somebody = true;
969 }
970
971 /*
972 * Signal that the process isn't on the wait list anymore. This allows
973 * LWLockDequeueSelf() to remove itself of the waitlist with a
974 * proclist_delete(), rather than having to check if it has been
975 * removed from the list.
976 */
977 Assert(waiter->lwWaiting == LW_WS_WAITING);
978 waiter->lwWaiting = LW_WS_PENDING_WAKEUP;
979
980 /*
981 * Once we've woken up an exclusive lock, there's no point in waking
982 * up anybody else.
983 */
984 if (waiter->lwWaitMode == LW_EXCLUSIVE)
985 break;
986 }
987
988 Assert(proclist_is_empty(&wakeup) || pg_atomic_read_u32(&lock->state) & LW_FLAG_HAS_WAITERS);
989
990 /* unset required flags, and release lock, in one fell swoop */
991 {
992 uint32 old_state;
993 uint32 desired_state;
994
995 old_state = pg_atomic_read_u32(&lock->state);
996 while (true)
997 {
998 desired_state = old_state;
999
1000 /* compute desired flags */
1001
1002 if (new_release_ok)
1003 desired_state |= LW_FLAG_RELEASE_OK;
1004 else
1005 desired_state &= ~LW_FLAG_RELEASE_OK;
1006
1007 if (proclist_is_empty(&lock->waiters))
1008 desired_state &= ~LW_FLAG_HAS_WAITERS;
1009
1010 desired_state &= ~LW_FLAG_LOCKED; /* release lock */
1011
1012 if (pg_atomic_compare_exchange_u32(&lock->state, &old_state,
1013 desired_state))
1014 break;
1015 }
1016 }
1017
1018 /* Awaken any waiters I removed from the queue. */
1019 proclist_foreach_modify(iter, &wakeup, lwWaitLink)
1020 {
1021 PGPROC *waiter = GetPGProcByNumber(iter.cur);
1022
1023 LOG_LWDEBUG("LWLockRelease", lock, "release waiter");
1024 proclist_delete(&wakeup, iter.cur, lwWaitLink);
1025
1026 /*
1027 * Guarantee that lwWaiting being unset only becomes visible once the
1028 * unlink from the link has completed. Otherwise the target backend
1029 * could be woken up for other reason and enqueue for a new lock - if
1030 * that happens before the list unlink happens, the list would end up
1031 * being corrupted.
1032 *
1033 * The barrier pairs with the LWLockWaitListLock() when enqueuing for
1034 * another lock.
1035 */
1036 pg_write_barrier();
1037 waiter->lwWaiting = LW_WS_NOT_WAITING;
1038 PGSemaphoreUnlock(waiter->sem);
1039 }
1040 }
1041
1042 /*
1043 * Add ourselves to the end of the queue.
1044 *
1045 * NB: Mode can be LW_WAIT_UNTIL_FREE here!
1046 */
1047 static void
1048 LWLockQueueSelf(LWLock *lock, LWLockMode mode)
1049 {
1050 /*
1051 * If we don't have a PGPROC structure, there's no way to wait. This
1052 * should never occur, since MyProc should only be null during shared
1053 * memory initialization.
1054 */
1055 if (MyProc == NULL)
1056 elog(PANIC, "cannot wait without a PGPROC structure");
1057
1058 if (MyProc->lwWaiting != LW_WS_NOT_WAITING)
1059 elog(PANIC, "queueing for lock while waiting on another one");
1060
1061 LWLockWaitListLock(lock);
1062
1063 /* setting the flag is protected by the spinlock */
1064 pg_atomic_fetch_or_u32(&lock->state, LW_FLAG_HAS_WAITERS);
1065
1066 MyProc->lwWaiting = LW_WS_WAITING;
1067 MyProc->lwWaitMode = mode;
1068
1069 /* LW_WAIT_UNTIL_FREE waiters are always at the front of the queue */
1070 if (mode == LW_WAIT_UNTIL_FREE)
1071 proclist_push_head(&lock->waiters, MyProcNumber, lwWaitLink);
1072 else
1073 proclist_push_tail(&lock->waiters, MyProcNumber, lwWaitLink);
1074
1075 /* Can release the mutex now */
1076 LWLockWaitListUnlock(lock);
1077
1078 #ifdef LOCK_DEBUG
1079 pg_atomic_fetch_add_u32(&lock->nwaiters, 1);
1080 #endif
1081 }
1082
1083 /*
1084 * Remove ourselves from the waitlist.
1085 *
1086 * This is used if we queued ourselves because we thought we needed to sleep
1087 * but, after further checking, we discovered that we don't actually need to
1088 * do so.
1089 */
1090 static void
1091 LWLockDequeueSelf(LWLock *lock)
1092 {
1093 bool on_waitlist;
1094
1095 #ifdef LWLOCK_STATS
1096 lwlock_stats *lwstats;
1097
1098 lwstats = get_lwlock_stats_entry(lock);
1099
1100 lwstats->dequeue_self_count++;
1101 #endif
1102
1103 LWLockWaitListLock(lock);
1104
1105 /*
1106 * Remove ourselves from the waitlist, unless we've already been removed.
1107 * The removal happens with the wait list lock held, so there's no race in
1108 * this check.
1109 */
1110 on_waitlist = MyProc->lwWaiting == LW_WS_WAITING;
1111 if (on_waitlist)
1112 proclist_delete(&lock->waiters, MyProcNumber, lwWaitLink);
1113
1114 if (proclist_is_empty(&lock->waiters) &&
1115 (pg_atomic_read_u32(&lock->state) & LW_FLAG_HAS_WAITERS) != 0)
1116 {
1117 pg_atomic_fetch_and_u32(&lock->state, ~LW_FLAG_HAS_WAITERS);
1118 }
1119
1120 /* XXX: combine with fetch_and above? */
1121 LWLockWaitListUnlock(lock);
1122
1123 /* clear waiting state again, nice for debugging */
1124 if (on_waitlist)
1125 MyProc->lwWaiting = LW_WS_NOT_WAITING;
1126 else
1127 {
1128 int extraWaits = 0;
1129
1130 /*
1131 * Somebody else dequeued us and has or will wake us up. Deal with the
1132 * superfluous absorption of a wakeup.
1133 */
1134
1135 /*
1136 * Reset RELEASE_OK flag if somebody woke us before we removed
1137 * ourselves - they'll have set it to false.
1138 */
1139 pg_atomic_fetch_or_u32(&lock->state, LW_FLAG_RELEASE_OK);
1140
1141 /*
1142 * Now wait for the scheduled wakeup, otherwise our ->lwWaiting would
1143 * get reset at some inconvenient point later. Most of the time this
1144 * will immediately return.
1145 */
1146 for (;;)
1147 {
1148 PGSemaphoreLock(MyProc->sem);
1149 if (MyProc->lwWaiting == LW_WS_NOT_WAITING)
1150 break;
1151 extraWaits++;
1152 }
1153
1154 /*
1155 * Fix the process wait semaphore's count for any absorbed wakeups.
1156 */
1157 while (extraWaits-- > 0)
1158 PGSemaphoreUnlock(MyProc->sem);
1159 }
1160
1161 #ifdef LOCK_DEBUG
1162 {
1163 /* not waiting anymore */
1164 uint32 nwaiters PG_USED_FOR_ASSERTS_ONLY = pg_atomic_fetch_sub_u32(&lock->nwaiters, 1);
1165
1166 Assert(nwaiters < MAX_BACKENDS);
1167 }
1168 #endif
1169 }
1170
1171 /*
1172 * LWLockAcquire - acquire a lightweight lock in the specified mode
1173 *
1174 * If the lock is not available, sleep until it is. Returns true if the lock
1175 * was available immediately, false if we had to sleep.
1176 *
1177 * Side effect: cancel/die interrupts are held off until lock release.
1178 */
1179 bool
1180 LWLockAcquire(LWLock *lock, LWLockMode mode)
1181 {
1182 PGPROC *proc = MyProc;
1183 bool result = true;
1184 int extraWaits = 0;
1185 #ifdef LWLOCK_STATS
1186 lwlock_stats *lwstats;
1187
1188 lwstats = get_lwlock_stats_entry(lock);
1189 #endif
1190
1191 Assert(mode == LW_SHARED || mode == LW_EXCLUSIVE);
1192
1193 PRINT_LWDEBUG("LWLockAcquire", lock, mode);
1194
1195 #ifdef LWLOCK_STATS
1196 /* Count lock acquisition attempts */
1197 if (mode == LW_EXCLUSIVE)
1198 lwstats->ex_acquire_count++;
1199 else
1200 lwstats->sh_acquire_count++;
1201 #endif /* LWLOCK_STATS */
1202
1203 /*
1204 * We can't wait if we haven't got a PGPROC. This should only occur
1205 * during bootstrap or shared memory initialization. Put an Assert here
1206 * to catch unsafe coding practices.
1207 */
1208 Assert(!(proc == NULL && IsUnderPostmaster));
1209
1210 /* Ensure we will have room to remember the lock */
1211 if (num_held_lwlocks >= MAX_SIMUL_LWLOCKS)
1212 elog(ERROR, "too many LWLocks taken");
1213
1214 /*
1215 * Lock out cancel/die interrupts until we exit the code section protected
1216 * by the LWLock. This ensures that interrupts will not interfere with
1217 * manipulations of data structures in shared memory.
1218 */
1219 HOLD_INTERRUPTS();
1220
1221 /*
1222 * Loop here to try to acquire lock after each time we are signaled by
1223 * LWLockRelease.
1224 *
1225 * NOTE: it might seem better to have LWLockRelease actually grant us the
1226 * lock, rather than retrying and possibly having to go back to sleep. But
1227 * in practice that is no good because it means a process swap for every
1228 * lock acquisition when two or more processes are contending for the same
1229 * lock. Since LWLocks are normally used to protect not-very-long
1230 * sections of computation, a process needs to be able to acquire and
1231 * release the same lock many times during a single CPU time slice, even
1232 * in the presence of contention. The efficiency of being able to do that
1233 * outweighs the inefficiency of sometimes wasting a process dispatch
1234 * cycle because the lock is not free when a released waiter finally gets
1235 * to run. See pgsql-hackers archives for 29-Dec-01.
1236 */
1237 for (;;)
1238 {
1239 bool mustwait;
1240
1241 /*
1242 * Try to grab the lock the first time, we're not in the waitqueue
1243 * yet/anymore.
1244 */
1245 mustwait = LWLockAttemptLock(lock, mode);
1246
1247 if (!mustwait)
1248 {
1249 LOG_LWDEBUG("LWLockAcquire", lock, "immediately acquired lock");
1250 break; /* got the lock */
1251 }
1252
1253 /*
1254 * Ok, at this point we couldn't grab the lock on the first try. We
1255 * cannot simply queue ourselves to the end of the list and wait to be
1256 * woken up because by now the lock could long have been released.
1257 * Instead add us to the queue and try to grab the lock again. If we
1258 * succeed we need to revert the queuing and be happy, otherwise we
1259 * recheck the lock. If we still couldn't grab it, we know that the
1260 * other locker will see our queue entries when releasing since they
1261 * existed before we checked for the lock.
1262 */
1263
1264 /* add to the queue */
1265 LWLockQueueSelf(lock, mode);
1266
1267 /* we're now guaranteed to be woken up if necessary */
1268 mustwait = LWLockAttemptLock(lock, mode);
1269
1270 /* ok, grabbed the lock the second time round, need to undo queueing */
1271 if (!mustwait)
1272 {
1273 LOG_LWDEBUG("LWLockAcquire", lock, "acquired, undoing queue");
1274
1275 LWLockDequeueSelf(lock);
1276 break;
1277 }
1278
1279 /*
1280 * Wait until awakened.
1281 *
1282 * It is possible that we get awakened for a reason other than being
1283 * signaled by LWLockRelease. If so, loop back and wait again. Once
1284 * we've gotten the LWLock, re-increment the sema by the number of
1285 * additional signals received.
1286 */
1287 LOG_LWDEBUG("LWLockAcquire", lock, "waiting");
1288
1289 #ifdef LWLOCK_STATS
1290 lwstats->block_count++;
1291 #endif
1292
1293 LWLockReportWaitStart(lock);
1294 if (TRACE_POSTGRESQL_LWLOCK_WAIT_START_ENABLED())
1295 TRACE_POSTGRESQL_LWLOCK_WAIT_START(T_NAME(lock), mode);
1296
1297 for (;;)
1298 {
1299 PGSemaphoreLock(proc->sem);
1300 if (proc->lwWaiting == LW_WS_NOT_WAITING)
1301 break;
1302 extraWaits++;
1303 }
1304
1305 /* Retrying, allow LWLockRelease to release waiters again. */
1306 pg_atomic_fetch_or_u32(&lock->state, LW_FLAG_RELEASE_OK);
1307
1308 #ifdef LOCK_DEBUG
1309 {
1310 /* not waiting anymore */
1311 uint32 nwaiters PG_USED_FOR_ASSERTS_ONLY = pg_atomic_fetch_sub_u32(&lock->nwaiters, 1);
1312
1313 Assert(nwaiters < MAX_BACKENDS);
1314 }
1315 #endif
1316
1317 if (TRACE_POSTGRESQL_LWLOCK_WAIT_DONE_ENABLED())
1318 TRACE_POSTGRESQL_LWLOCK_WAIT_DONE(T_NAME(lock), mode);
1319 LWLockReportWaitEnd();
1320
1321 LOG_LWDEBUG("LWLockAcquire", lock, "awakened");
1322
1323 /* Now loop back and try to acquire lock again. */
1324 result = false;
1325 }
1326
1327 if (TRACE_POSTGRESQL_LWLOCK_ACQUIRE_ENABLED())
1328 TRACE_POSTGRESQL_LWLOCK_ACQUIRE(T_NAME(lock), mode);
1329
1330 /* Add lock to list of locks held by this backend */
1331 held_lwlocks[num_held_lwlocks].lock = lock;
1332 held_lwlocks[num_held_lwlocks++].mode = mode;
1333
1334 /*
1335 * Fix the process wait semaphore's count for any absorbed wakeups.
1336 */
1337 while (extraWaits-- > 0)
1338 PGSemaphoreUnlock(proc->sem);
1339
1340 return result;
1341 }
1342
1343 /*
1344 * LWLockConditionalAcquire - acquire a lightweight lock in the specified mode
1345 *
1346 * If the lock is not available, return false with no side-effects.
1347 *
1348 * If successful, cancel/die interrupts are held off until lock release.
1349 */
1350 bool
1351 LWLockConditionalAcquire(LWLock *lock, LWLockMode mode)
1352 {
1353 bool mustwait;
1354
1355 Assert(mode == LW_SHARED || mode == LW_EXCLUSIVE);
1356
1357 PRINT_LWDEBUG("LWLockConditionalAcquire", lock, mode);
1358
1359 /* Ensure we will have room to remember the lock */
1360 if (num_held_lwlocks >= MAX_SIMUL_LWLOCKS)
1361 elog(ERROR, "too many LWLocks taken");
1362
1363 /*
1364 * Lock out cancel/die interrupts until we exit the code section protected
1365 * by the LWLock. This ensures that interrupts will not interfere with
1366 * manipulations of data structures in shared memory.
1367 */
1368 HOLD_INTERRUPTS();
1369
1370 /* Check for the lock */
1371 mustwait = LWLockAttemptLock(lock, mode);
1372
1373 if (mustwait)
1374 {
1375 /* Failed to get lock, so release interrupt holdoff */
1376 RESUME_INTERRUPTS();
1377
1378 LOG_LWDEBUG("LWLockConditionalAcquire", lock, "failed");
1379 if (TRACE_POSTGRESQL_LWLOCK_CONDACQUIRE_FAIL_ENABLED())
1380 TRACE_POSTGRESQL_LWLOCK_CONDACQUIRE_FAIL(T_NAME(lock), mode);
1381 }
1382 else
1383 {
1384 /* Add lock to list of locks held by this backend */
1385 held_lwlocks[num_held_lwlocks].lock = lock;
1386 held_lwlocks[num_held_lwlocks++].mode = mode;
1387 if (TRACE_POSTGRESQL_LWLOCK_CONDACQUIRE_ENABLED())
1388 TRACE_POSTGRESQL_LWLOCK_CONDACQUIRE(T_NAME(lock), mode);
1389 }
1390 return !mustwait;
1391 }
1392
1393 /*
1394 * LWLockAcquireOrWait - Acquire lock, or wait until it's free
1395 *
1396 * The semantics of this function are a bit funky. If the lock is currently
1397 * free, it is acquired in the given mode, and the function returns true. If
1398 * the lock isn't immediately free, the function waits until it is released
1399 * and returns false, but does not acquire the lock.
1400 *
1401 * This is currently used for WALWriteLock: when a backend flushes the WAL,
1402 * holding WALWriteLock, it can flush the commit records of many other
1403 * backends as a side-effect. Those other backends need to wait until the
1404 * flush finishes, but don't need to acquire the lock anymore. They can just
1405 * wake up, observe that their records have already been flushed, and return.
1406 */
1407 bool
1408 LWLockAcquireOrWait(LWLock *lock, LWLockMode mode)
1409 {
1410 PGPROC *proc = MyProc;
1411 bool mustwait;
1412 int extraWaits = 0;
1413 #ifdef LWLOCK_STATS
1414 lwlock_stats *lwstats;
1415
1416 lwstats = get_lwlock_stats_entry(lock);
1417 #endif
1418
1419 Assert(mode == LW_SHARED || mode == LW_EXCLUSIVE);
1420
1421 PRINT_LWDEBUG("LWLockAcquireOrWait", lock, mode);
1422
1423 /* Ensure we will have room to remember the lock */
1424 if (num_held_lwlocks >= MAX_SIMUL_LWLOCKS)
1425 elog(ERROR, "too many LWLocks taken");
1426
1427 /*
1428 * Lock out cancel/die interrupts until we exit the code section protected
1429 * by the LWLock. This ensures that interrupts will not interfere with
1430 * manipulations of data structures in shared memory.
1431 */
1432 HOLD_INTERRUPTS();
1433
1434 /*
1435 * NB: We're using nearly the same twice-in-a-row lock acquisition
1436 * protocol as LWLockAcquire(). Check its comments for details.
1437 */
1438 mustwait = LWLockAttemptLock(lock, mode);
1439
1440 if (mustwait)
1441 {
1442 LWLockQueueSelf(lock, LW_WAIT_UNTIL_FREE);
1443
1444 mustwait = LWLockAttemptLock(lock, mode);
1445
1446 if (mustwait)
1447 {
1448 /*
1449 * Wait until awakened. Like in LWLockAcquire, be prepared for
1450 * bogus wakeups.
1451 */
1452 LOG_LWDEBUG("LWLockAcquireOrWait", lock, "waiting");
1453
1454 #ifdef LWLOCK_STATS
1455 lwstats->block_count++;
1456 #endif
1457
1458 LWLockReportWaitStart(lock);
1459 if (TRACE_POSTGRESQL_LWLOCK_WAIT_START_ENABLED())
1460 TRACE_POSTGRESQL_LWLOCK_WAIT_START(T_NAME(lock), mode);
1461
1462 for (;;)
1463 {
1464 PGSemaphoreLock(proc->sem);
1465 if (proc->lwWaiting == LW_WS_NOT_WAITING)
1466 break;
1467 extraWaits++;
1468 }
1469
1470 #ifdef LOCK_DEBUG
1471 {
1472 /* not waiting anymore */
1473 uint32 nwaiters PG_USED_FOR_ASSERTS_ONLY = pg_atomic_fetch_sub_u32(&lock->nwaiters, 1);
1474
1475 Assert(nwaiters < MAX_BACKENDS);
1476 }
1477 #endif
1478 if (TRACE_POSTGRESQL_LWLOCK_WAIT_DONE_ENABLED())
1479 TRACE_POSTGRESQL_LWLOCK_WAIT_DONE(T_NAME(lock), mode);
1480 LWLockReportWaitEnd();
1481
1482 LOG_LWDEBUG("LWLockAcquireOrWait", lock, "awakened");
1483 }
1484 else
1485 {
1486 LOG_LWDEBUG("LWLockAcquireOrWait", lock, "acquired, undoing queue");
1487
1488 /*
1489 * Got lock in the second attempt, undo queueing. We need to treat
1490 * this as having successfully acquired the lock, otherwise we'd
1491 * not necessarily wake up people we've prevented from acquiring
1492 * the lock.
1493 */
1494 LWLockDequeueSelf(lock);
1495 }
1496 }
1497
1498 /*
1499 * Fix the process wait semaphore's count for any absorbed wakeups.
1500 */
1501 while (extraWaits-- > 0)
1502 PGSemaphoreUnlock(proc->sem);
1503
1504 if (mustwait)
1505 {
1506 /* Failed to get lock, so release interrupt holdoff */
1507 RESUME_INTERRUPTS();
1508 LOG_LWDEBUG("LWLockAcquireOrWait", lock, "failed");
1509 if (TRACE_POSTGRESQL_LWLOCK_ACQUIRE_OR_WAIT_FAIL_ENABLED())
1510 TRACE_POSTGRESQL_LWLOCK_ACQUIRE_OR_WAIT_FAIL(T_NAME(lock), mode);
1511 }
1512 else
1513 {
1514 LOG_LWDEBUG("LWLockAcquireOrWait", lock, "succeeded");
1515 /* Add lock to list of locks held by this backend */
1516 held_lwlocks[num_held_lwlocks].lock = lock;
1517 held_lwlocks[num_held_lwlocks++].mode = mode;
1518 if (TRACE_POSTGRESQL_LWLOCK_ACQUIRE_OR_WAIT_ENABLED())
1519 TRACE_POSTGRESQL_LWLOCK_ACQUIRE_OR_WAIT(T_NAME(lock), mode);
1520 }
1521
1522 return !mustwait;
1523 }
1524
1525 /*
1526 * Does the lwlock in its current state need to wait for the variable value to
1527 * change?
1528 *
1529 * If we don't need to wait, and it's because the value of the variable has
1530 * changed, store the current value in newval.
1531 *
1532 * *result is set to true if the lock was free, and false otherwise.
1533 */
1534 static bool
1535 LWLockConflictsWithVar(LWLock *lock, pg_atomic_uint64 *valptr, uint64 oldval,
1536 uint64 *newval, bool *result)
1537 {
1538 bool mustwait;
1539 uint64 value;
1540
1541 /*
1542 * Test first to see if it the slot is free right now.
1543 *
1544 * XXX: the unique caller of this routine, WaitXLogInsertionsToFinish()
1545 * via LWLockWaitForVar(), uses an implied barrier with a spinlock before
1546 * this, so we don't need a memory barrier here as far as the current
1547 * usage is concerned. But that might not be safe in general.
1548 */
1549 mustwait = (pg_atomic_read_u32(&lock->state) & LW_VAL_EXCLUSIVE) != 0;
1550
1551 if (!mustwait)
1552 {
1553 *result = true;
1554 return false;
1555 }
1556
1557 *result = false;
1558
1559 /*
1560 * Reading this value atomically is safe even on platforms where uint64
1561 * cannot be read without observing a torn value.
1562 */
1563 value = pg_atomic_read_u64(valptr);
1564
1565 if (value != oldval)
1566 {
1567 mustwait = false;
1568 *newval = value;
1569 }
1570 else
1571 {
1572 mustwait = true;
1573 }
1574
1575 return mustwait;
1576 }
1577
1578 /*
1579 * LWLockWaitForVar - Wait until lock is free, or a variable is updated.
1580 *
1581 * If the lock is held and *valptr equals oldval, waits until the lock is
1582 * either freed, or the lock holder updates *valptr by calling
1583 * LWLockUpdateVar. If the lock is free on exit (immediately or after
1584 * waiting), returns true. If the lock is still held, but *valptr no longer
1585 * matches oldval, returns false and sets *newval to the current value in
1586 * *valptr.
1587 *
1588 * Note: this function ignores shared lock holders; if the lock is held
1589 * in shared mode, returns 'true'.
1590 *
1591 * Be aware that LWLockConflictsWithVar() does not include a memory barrier,
1592 * hence the caller of this function may want to rely on an explicit barrier or
1593 * an implied barrier via spinlock or LWLock to avoid memory ordering issues.
1594 */
1595 bool
1596 LWLockWaitForVar(LWLock *lock, pg_atomic_uint64 *valptr, uint64 oldval,
1597 uint64 *newval)
1598 {
1599 PGPROC *proc = MyProc;
1600 int extraWaits = 0;
1601 bool result = false;
1602 #ifdef LWLOCK_STATS
1603 lwlock_stats *lwstats;
1604
1605 lwstats = get_lwlock_stats_entry(lock);
1606 #endif
1607
1608 PRINT_LWDEBUG("LWLockWaitForVar", lock, LW_WAIT_UNTIL_FREE);
1609
1610 /*
1611 * Lock out cancel/die interrupts while we sleep on the lock. There is no
1612 * cleanup mechanism to remove us from the wait queue if we got
1613 * interrupted.
1614 */
1615 HOLD_INTERRUPTS();
1616
1617 /*
1618 * Loop here to check the lock's status after each time we are signaled.
1619 */
1620 for (;;)
1621 {
1622 bool mustwait;
1623
1624 mustwait = LWLockConflictsWithVar(lock, valptr, oldval, newval,
1625 &result);
1626
1627 if (!mustwait)
1628 break; /* the lock was free or value didn't match */
1629
1630 /*
1631 * Add myself to wait queue. Note that this is racy, somebody else
1632 * could wakeup before we're finished queuing. NB: We're using nearly
1633 * the same twice-in-a-row lock acquisition protocol as
1634 * LWLockAcquire(). Check its comments for details. The only
1635 * difference is that we also have to check the variable's values when
1636 * checking the state of the lock.
1637 */
1638 LWLockQueueSelf(lock, LW_WAIT_UNTIL_FREE);
1639
1640 /*
1641 * Set RELEASE_OK flag, to make sure we get woken up as soon as the
1642 * lock is released.
1643 */
1644 pg_atomic_fetch_or_u32(&lock->state, LW_FLAG_RELEASE_OK);
1645
1646 /*
1647 * We're now guaranteed to be woken up if necessary. Recheck the lock
1648 * and variables state.
1649 */
1650 mustwait = LWLockConflictsWithVar(lock, valptr, oldval, newval,
1651 &result);
1652
1653 /* Ok, no conflict after we queued ourselves. Undo queueing. */
1654 if (!mustwait)
1655 {
1656 LOG_LWDEBUG("LWLockWaitForVar", lock, "free, undoing queue");
1657
1658 LWLockDequeueSelf(lock);
1659 break;
1660 }
1661
1662 /*
1663 * Wait until awakened.
1664 *
1665 * It is possible that we get awakened for a reason other than being
1666 * signaled by LWLockRelease. If so, loop back and wait again. Once
1667 * we've gotten the LWLock, re-increment the sema by the number of
1668 * additional signals received.
1669 */
1670 LOG_LWDEBUG("LWLockWaitForVar", lock, "waiting");
1671
1672 #ifdef LWLOCK_STATS
1673 lwstats->block_count++;
1674 #endif
1675
1676 LWLockReportWaitStart(lock);
1677 if (TRACE_POSTGRESQL_LWLOCK_WAIT_START_ENABLED())
1678 TRACE_POSTGRESQL_LWLOCK_WAIT_START(T_NAME(lock), LW_EXCLUSIVE);
1679
1680 for (;;)
1681 {
1682 PGSemaphoreLock(proc->sem);
1683 if (proc->lwWaiting == LW_WS_NOT_WAITING)
1684 break;
1685 extraWaits++;
1686 }
1687
1688 #ifdef LOCK_DEBUG
1689 {
1690 /* not waiting anymore */
1691 uint32 nwaiters PG_USED_FOR_ASSERTS_ONLY = pg_atomic_fetch_sub_u32(&lock->nwaiters, 1);
1692
1693 Assert(nwaiters < MAX_BACKENDS);
1694 }
1695 #endif
1696
1697 if (TRACE_POSTGRESQL_LWLOCK_WAIT_DONE_ENABLED())
1698 TRACE_POSTGRESQL_LWLOCK_WAIT_DONE(T_NAME(lock), LW_EXCLUSIVE);
1699 LWLockReportWaitEnd();
1700
1701 LOG_LWDEBUG("LWLockWaitForVar", lock, "awakened");
1702
1703 /* Now loop back and check the status of the lock again. */
1704 }
1705
1706 /*
1707 * Fix the process wait semaphore's count for any absorbed wakeups.
1708 */
1709 while (extraWaits-- > 0)
1710 PGSemaphoreUnlock(proc->sem);
1711
1712 /*
1713 * Now okay to allow cancel/die interrupts.
1714 */
1715 RESUME_INTERRUPTS();
1716
1717 return result;
1718 }
1719
1720
1721 /*
1722 * LWLockUpdateVar - Update a variable and wake up waiters atomically
1723 *
1724 * Sets *valptr to 'val', and wakes up all processes waiting for us with
1725 * LWLockWaitForVar(). It first sets the value atomically and then wakes up
1726 * waiting processes so that any process calling LWLockWaitForVar() on the same
1727 * lock is guaranteed to see the new value, and act accordingly.
1728 *
1729 * The caller must be holding the lock in exclusive mode.
1730 */
1731 void
1732 LWLockUpdateVar(LWLock *lock, pg_atomic_uint64 *valptr, uint64 val)
1733 {
1734 proclist_head wakeup;
1735 proclist_mutable_iter iter;
1736
1737 PRINT_LWDEBUG("LWLockUpdateVar", lock, LW_EXCLUSIVE);
1738
1739 /*
1740 * Note that pg_atomic_exchange_u64 is a full barrier, so we're guaranteed
1741 * that the variable is updated before waking up waiters.
1742 */
1743 pg_atomic_exchange_u64(valptr, val);
1744
1745 proclist_init(&wakeup);
1746
1747 LWLockWaitListLock(lock);
1748
1749 Assert(pg_atomic_read_u32(&lock->state) & LW_VAL_EXCLUSIVE);
1750
1751 /*
1752 * See if there are any LW_WAIT_UNTIL_FREE waiters that need to be woken
1753 * up. They are always in the front of the queue.
1754 */
1755 proclist_foreach_modify(iter, &lock->waiters, lwWaitLink)
1756 {
1757 PGPROC *waiter = GetPGProcByNumber(iter.cur);
1758
1759 if (waiter->lwWaitMode != LW_WAIT_UNTIL_FREE)
1760 break;
1761
1762 proclist_delete(&lock->waiters, iter.cur, lwWaitLink);
1763 proclist_push_tail(&wakeup, iter.cur, lwWaitLink);
1764
1765 /* see LWLockWakeup() */
1766 Assert(waiter->lwWaiting == LW_WS_WAITING);
1767 waiter->lwWaiting = LW_WS_PENDING_WAKEUP;
1768 }
1769
1770 /* We are done updating shared state of the lock itself. */
1771 LWLockWaitListUnlock(lock);
1772
1773 /*
1774 * Awaken any waiters I removed from the queue.
1775 */
1776 proclist_foreach_modify(iter, &wakeup, lwWaitLink)
1777 {
1778 PGPROC *waiter = GetPGProcByNumber(iter.cur);
1779
1780 proclist_delete(&wakeup, iter.cur, lwWaitLink);
1781 /* check comment in LWLockWakeup() about this barrier */
1782 pg_write_barrier();
1783 waiter->lwWaiting = LW_WS_NOT_WAITING;
1784 PGSemaphoreUnlock(waiter->sem);
1785 }
1786 }
1787
1788
1789 /*
1790 * Stop treating lock as held by current backend.
1791 *
1792 * This is the code that can be shared between actually releasing a lock
1793 * (LWLockRelease()) and just not tracking ownership of the lock anymore
1794 * without releasing the lock (LWLockDisown()).
1795 *
1796 * Returns the mode in which the lock was held by the current backend.
1797 *
1798 * NB: This does not call RESUME_INTERRUPTS(), but leaves that responsibility
1799 * of the caller.
1800 *
1801 * NB: This will leave lock->owner pointing to the current backend (if
1802 * LOCK_DEBUG is set). This is somewhat intentional, as it makes it easier to
1803 * debug cases of missing wakeups during lock release.
1804 */
1805 static inline LWLockMode
1806 LWLockDisownInternal(LWLock *lock)
1807 {
1808 LWLockMode mode;
1809 int i;
1810
1811 /*
1812 * Remove lock from list of locks held. Usually, but not always, it will
1813 * be the latest-acquired lock; so search array backwards.
1814 */
1815 for (i = num_held_lwlocks; --i >= 0;)
1816 if (lock == held_lwlocks[i].lock)
1817 break;
1818
1819 if (i < 0)
1820 elog(ERROR, "lock %s is not held", T_NAME(lock));
1821
1822 mode = held_lwlocks[i].mode;
1823
1824 num_held_lwlocks--;
1825 for (; i < num_held_lwlocks; i++)
1826 held_lwlocks[i] = held_lwlocks[i + 1];
1827
1828 return mode;
1829 }
1830
1831 /*
1832 * Helper function to release lock, shared between LWLockRelease() and
1833 * LWLockReleaseDisowned().
1834 */
1835 static void
1836 LWLockReleaseInternal(LWLock *lock, LWLockMode mode)
1837 {
1838 uint32 oldstate;
1839 bool check_waiters;
1840
1841 /*
1842 * Release my hold on lock, after that it can immediately be acquired by
1843 * others, even if we still have to wakeup other waiters.
1844 */
1845 if (mode == LW_EXCLUSIVE)
1846 oldstate = pg_atomic_sub_fetch_u32(&lock->state, LW_VAL_EXCLUSIVE);
1847 else
1848 oldstate = pg_atomic_sub_fetch_u32(&lock->state, LW_VAL_SHARED);
1849
1850 /* nobody else can have that kind of lock */
1851 Assert(!(oldstate & LW_VAL_EXCLUSIVE));
1852
1853 if (TRACE_POSTGRESQL_LWLOCK_RELEASE_ENABLED())
1854 TRACE_POSTGRESQL_LWLOCK_RELEASE(T_NAME(lock));
1855
1856 /*
1857 * We're still waiting for backends to get scheduled, don't wake them up
1858 * again.
1859 */
1860 if ((oldstate & (LW_FLAG_HAS_WAITERS | LW_FLAG_RELEASE_OK)) ==
1861 (LW_FLAG_HAS_WAITERS | LW_FLAG_RELEASE_OK) &&
1862 (oldstate & LW_LOCK_MASK) == 0)
1863 check_waiters = true;
1864 else
1865 check_waiters = false;
1866
1867 /*
1868 * As waking up waiters requires the spinlock to be acquired, only do so
1869 * if necessary.
1870 */
1871 if (check_waiters)
1872 {
1873 /* XXX: remove before commit? */
1874 LOG_LWDEBUG("LWLockRelease", lock, "releasing waiters");
1875 LWLockWakeup(lock);
1876 }
1877 }
1878
1879
1880 /*
1881 * Stop treating lock as held by current backend.
1882 *
1883 * After calling this function it's the callers responsibility to ensure that
1884 * the lock gets released (via LWLockReleaseDisowned()), even in case of an
1885 * error. This only is desirable if the lock is going to be released in a
1886 * different process than the process that acquired it.
1887 */
1888 void
1889 LWLockDisown(LWLock *lock)
1890 {
1891 LWLockDisownInternal(lock);
1892
1893 RESUME_INTERRUPTS();
1894 }
1895
1896 /*
1897 * LWLockRelease - release a previously acquired lock
1898 */
1899 void
1900 LWLockRelease(LWLock *lock)
1901 {
1902 LWLockMode mode;
1903
1904 mode = LWLockDisownInternal(lock);
1905
1906 PRINT_LWDEBUG("LWLockRelease", lock, mode);
1907
1908 LWLockReleaseInternal(lock, mode);
1909
1910 /*
1911 * Now okay to allow cancel/die interrupts.
1912 */
1913 RESUME_INTERRUPTS();
1914 }
1915
1916 /*
1917 * Release lock previously disowned with LWLockDisown().
1918 */
1919 void
1920 LWLockReleaseDisowned(LWLock *lock, LWLockMode mode)
1921 {
1922 LWLockReleaseInternal(lock, mode);
1923 }
1924
1925 /*
1926 * LWLockReleaseClearVar - release a previously acquired lock, reset variable
1927 */
1928 void
1929 LWLockReleaseClearVar(LWLock *lock, pg_atomic_uint64 *valptr, uint64 val)
1930 {
1931 /*
1932 * Note that pg_atomic_exchange_u64 is a full barrier, so we're guaranteed
1933 * that the variable is updated before releasing the lock.
1934 */
1935 pg_atomic_exchange_u64(valptr, val);
1936
1937 LWLockRelease(lock);
1938 }
1939
1940
1941 /*
1942 * LWLockReleaseAll - release all currently-held locks
1943 *
1944 * Used to clean up after ereport(ERROR). An important difference between this
1945 * function and retail LWLockRelease calls is that InterruptHoldoffCount is
1946 * unchanged by this operation. This is necessary since InterruptHoldoffCount
1947 * has been set to an appropriate level earlier in error recovery. We could
1948 * decrement it below zero if we allow it to drop for each released lock!
1949 *
1950 * Note that this function must be safe to call even before the LWLock
1951 * subsystem has been initialized (e.g., during early startup failures).
1952 * In that case, num_held_lwlocks will be 0 and we do nothing.
1953 */
1954 void
1955 LWLockReleaseAll(void)
1956 {
1957 while (num_held_lwlocks > 0)
1958 {
1959 HOLD_INTERRUPTS(); /* match the upcoming RESUME_INTERRUPTS */
1960
1961 LWLockRelease(held_lwlocks[num_held_lwlocks - 1].lock);
1962 }
1963
1964 Assert(num_held_lwlocks == 0);
1965 }
1966
1967
1968 /*
1969 * ForEachLWLockHeldByMe - run a callback for each held lock
1970 *
1971 * This is meant as debug support only.
1972 */
1973 void
1974 ForEachLWLockHeldByMe(void (*callback) (LWLock *, LWLockMode, void *),
1975 void *context)
1976 {
1977 int i;
1978
1979 for (i = 0; i < num_held_lwlocks; i++)
1980 callback(held_lwlocks[i].lock, held_lwlocks[i].mode, context);
1981 }
1982
1983 /*
1984 * LWLockHeldByMe - test whether my process holds a lock in any mode
1985 *
1986 * This is meant as debug support only.
1987 */
1988 bool
1989 LWLockHeldByMe(LWLock *lock)
1990 {
1991 int i;
1992
1993 for (i = 0; i < num_held_lwlocks; i++)
1994 {
1995 if (held_lwlocks[i].lock == lock)
1996 return true;
1997 }
1998 return false;
1999 }
2000
2001 /*
2002 * LWLockAnyHeldByMe - test whether my process holds any of an array of locks
2003 *
2004 * This is meant as debug support only.
2005 */
2006 bool
2007 LWLockAnyHeldByMe(LWLock *lock, int nlocks, size_t stride)
2008 {
2009 char *held_lock_addr;
2010 char *begin;
2011 char *end;
2012 int i;
2013
2014 begin = (char *) lock;
2015 end = begin + nlocks * stride;
2016 for (i = 0; i < num_held_lwlocks; i++)
2017 {
2018 held_lock_addr = (char *) held_lwlocks[i].lock;
2019 if (held_lock_addr >= begin &&
2020 held_lock_addr < end &&
2021 (held_lock_addr - begin) % stride == 0)
2022 return true;
2023 }
2024 return false;
2025 }
2026
2027 /*
2028 * LWLockHeldByMeInMode - test whether my process holds a lock in given mode
2029 *
2030 * This is meant as debug support only.
2031 */
2032 bool
2033 LWLockHeldByMeInMode(LWLock *lock, LWLockMode mode)
2034 {
2035 int i;
2036
2037 for (i = 0; i < num_held_lwlocks; i++)
2038 {
2039 if (held_lwlocks[i].lock == lock && held_lwlocks[i].mode == mode)
2040 return true;
2041 }
2042 return false;
2043 }