Debian Codegraph @

dirsrvtests/tests/stress/backend/bdb_cursor_race_test.py

raw
1 # --- BEGIN COPYRIGHT BLOCK ---
2 # Copyright (C) 2025 Red Hat, Inc.
3 # All rights reserved.
4 #
5 # License: GPL (version 3 or any later version).
6 # See LICENSE for details.
7 # --- END COPYRIGHT BLOCK ---
8 #
9
10 import pytest
11 import logging
12 import ldap
13 import ldap.modlist as modlist
14 from threading import Thread, Event, Lock
15 import random
16 import time
17 from test389.topologies import topology_st
18 from lib389._constants import DEFAULT_SUFFIX, DN_DM, PW_DM
19 from lib389.idm.user import UserAccounts
20 from lib389.idm.organizationalunit import OrganizationalUnits
21
22 pytestmark = pytest.mark.tier2
23
24 log = logging.getLogger(__name__)
25
26 # Test configuration
27 SEARCH_THREADS = 30
28 MODIFY_THREADS = 15
29 DELETE_THREADS = 8
30 CHURN_THREADS = 5
31 DEFAULT_TIMEOUT = 60 * 60 # 60 minutes in seconds
32
33 # Statistics and crash detection
34 stats_lock = Lock()
35 stats = {
36 'searches': 0,
37 'modifies': 0,
38 'deletes': 0,
39 'adds': 0,
40 'errors': 0,
41 'start_time': None
42 }
43
44 # Global flag to track if server crashed
45 server_crashed = False
46 crash_lock = Lock()
47
48 HIGH_CONTENTION_FILTERS = [
49 "(objectClass=inetOrgPerson)",
50 "(objectClass=person)",
51 "(objectClass=*)",
52 "(uid=*)",
53 "(cn=*)",
54 "(mail=*)",
55 "(description=*)",
56 "(telephoneNumber=*)",
57 ]
58
59
60 def create_raw_connection(host, port):
61 """Create a raw LDAP connection independent of the instance connection pool
62
63 :param host: LDAP server hostname
64 :param port: LDAP server port
65 :returns: Raw LDAP connection
66 """
67 # Use ldap.initialize to create a completely independent connection
68 uri = f"ldap://{host}:{port}"
69 conn = ldap.initialize(uri)
70 conn.protocol_version = ldap.VERSION3
71
72 # Short timeout to fail fast
73 conn.set_option(ldap.OPT_NETWORK_TIMEOUT, 5.0)
74 conn.set_option(ldap.OPT_TIMEOUT, 5.0)
75
76 # Bind as Directory Manager
77 conn.simple_bind_s(DN_DM, PW_DM)
78 return conn
79
80
81 def update_stats(operation, count=1):
82 """Update operation statistics in a thread-safe manner"""
83 with stats_lock:
84 stats[operation] = stats.get(operation, 0) + count
85
86
87 def mark_server_crashed():
88 """Mark that the server has crashed"""
89 global server_crashed
90 with crash_lock:
91 server_crashed = True
92
93
94 def print_stats():
95 """Print current statistics"""
96 with stats_lock:
97 if stats['start_time']:
98 elapsed = time.time() - stats['start_time']
99 log.info(f"Statistics after {elapsed:.1f}s: "
100 f"searches={stats['searches']}, "
101 f"modifies={stats['modifies']}, "
102 f"adds={stats['adds']}, "
103 f"deletes={stats['deletes']}, "
104 f"errors={stats['errors']}")
105
106
107 def setup_test_data(inst, num_entries=1000):
108 """Setup test data with multi-valued attributes
109
110 :param inst: DirSrv instance
111 :param num_entries: Number of test entries to create
112 """
113 log.info(f"Setting up {num_entries} test entries with multi-valued attributes...")
114
115 # Create entries with many multi-valued attributes using raw LDAP
116 created = 0
117 for i in range(num_entries):
118 try:
119 dn = f"uid=test{i},ou=people,{DEFAULT_SUFFIX}"
120
121 descriptions = [f"description_{i}_{j}_{'x'*50}".encode() for j in range(20)]
122 phones = [f"+1-555-{i:04d}-{j:04d}".encode() for j in range(10)]
123 mails = [f"test{i}.{j}@example.com".encode() for j in range(5)]
124
125 attrs = {
126 'objectClass': [b'top', b'person', b'organizationalPerson', b'inetOrgPerson'],
127 'uid': [f'test{i}'.encode()],
128 'cn': [f'Test User {i}'.encode()],
129 'sn': [f'User{i}'.encode()],
130 'description': descriptions,
131 'telephoneNumber': phones,
132 'mail': mails,
133 'userPassword': [b'password'],
134 }
135
136 inst.add_s(dn, modlist.addModlist(attrs))
137 created += 1
138
139 if (i + 1) % 100 == 0:
140 log.info(f"Created {i + 1}/{num_entries} entries...")
141
142 except ldap.ALREADY_EXISTS:
143 pass
144 except Exception as e:
145 log.warning(f"Error creating entry {i}: {e}")
146
147 log.info(f"Test data setup complete. Created {created} new entries.")
148
149
150 def search_worker(stop_event, host, port, thread_id):
151 """Worker thread for high-contention searches
152
153 :param stop_event: Event to signal thread termination
154 :param host: LDAP server hostname
155 :param port: LDAP server port
156 :param thread_id: Thread identifier
157 """
158 # Create independent raw LDAP connection for this thread
159 conn = create_raw_connection(host, port)
160 log.info(f"Search worker {thread_id} started")
161 search_count = 0
162
163 try:
164 while not stop_event.is_set():
165 try:
166 filter_str = random.choice(HIGH_CONTENTION_FILTERS)
167 scope = random.choice([ldap.SCOPE_SUBTREE, ldap.SCOPE_ONELEVEL])
168
169 conn.search_s(
170 DEFAULT_SUFFIX,
171 scope,
172 filter_str,
173 attrlist=['dn']
174 )
175
176 search_count += 1
177
178 if search_count % 100 == 0:
179 update_stats('searches', 100)
180
181 except ldap.SERVER_DOWN:
182 log.error(f"Search worker {thread_id}: SERVER DOWN - ns-slapd crashed!")
183 mark_server_crashed()
184 stop_event.set()
185 break
186 except Exception as e:
187 update_stats('errors')
188 if search_count < 10:
189 log.debug(f"Search worker {thread_id} error: {e}")
190 finally:
191 try:
192 conn.unbind_s()
193 except:
194 pass
195 log.info(f"Search worker {thread_id} stopped (total searches: {search_count})")
196
197
198 def modify_worker(stop_event, host, port, thread_id):
199 """Worker thread for modifying multi-valued attributes
200
201 :param stop_event: Event to signal thread termination
202 :param host: LDAP server hostname
203 :param port: LDAP server port
204 :param thread_id: Thread identifier
205 """
206 # Create independent raw LDAP connection for this thread
207 conn = create_raw_connection(host, port)
208 log.info(f"Modify worker {thread_id} started")
209 modify_count = 0
210
211 try:
212 while not stop_event.is_set():
213 try:
214 entry_num = random.randint(0, 999)
215 dn = f"uid=test{entry_num},ou=people,{DEFAULT_SUFFIX}"
216 attr = 'description'
217 value = f"{attr}_{int(time.time() * 1000000)}_{'x'*100}"
218
219 # Strategy 1: Add a new value (increases index size)
220 if random.random() < 0.3:
221 mod_attrs = [(ldap.MOD_ADD, attr, [value.encode()])]
222
223 # Strategy 2: Replace all values (causes delete + add)
224 # This should trigger __db_ditem_nolog() in the delete phase
225 elif random.random() < 0.7:
226 new_values = [f"{attr}_new_{i}_{'y'*100}".encode() for i in range(20)]
227 mod_attrs = [(ldap.MOD_REPLACE, attr, new_values)]
228
229 # Strategy 3: Delete all values, then add multiple new ones
230 else:
231 new_values = [f"{attr}_churn_{i}_{'z'*100}".encode() for i in range(15)]
232 mod_attrs = [
233 (ldap.MOD_DELETE, attr, []),
234 (ldap.MOD_ADD, attr, new_values)
235 ]
236
237 conn.modify_s(dn, mod_attrs)
238 modify_count += 1
239
240 if modify_count % 50 == 0:
241 update_stats('modifies', 50)
242
243 except ldap.SERVER_DOWN:
244 log.error(f"Modify worker {thread_id}: SERVER DOWN - ns-slapd crashed!")
245 mark_server_crashed()
246 stop_event.set()
247 break
248 except ldap.NO_SUCH_OBJECT:
249 pass
250 except Exception as e:
251 update_stats('errors')
252 if modify_count < 10:
253 log.debug(f"Modify worker {thread_id} error: {e}")
254 finally:
255 try:
256 conn.unbind_s()
257 except:
258 pass
259 log.info(f"Modify worker {thread_id} stopped (total modifies: {modify_count})")
260
261
262 def delete_worker(stop_event, host, port, thread_id):
263 """Worker thread for deleting entries
264
265 :param stop_event: Event to signal thread termination
266 :param host: LDAP server hostname
267 :param port: LDAP server port
268 :param thread_id: Thread identifier
269 """
270 # Create independent raw LDAP connection for this thread
271 conn = create_raw_connection(host, port)
272 log.info(f"Delete worker {thread_id} started")
273 delete_count = 0
274 delete_range_start = 5000 + (thread_id * 1000)
275
276 try:
277 while not stop_event.is_set():
278 try:
279 entry_num = delete_range_start + (delete_count % 1000)
280 dn = f"uid=delete{entry_num},ou=people,{DEFAULT_SUFFIX}"
281
282 conn.delete_s(dn)
283 delete_count += 1
284
285 if delete_count % 20 == 0:
286 update_stats('deletes', 20)
287
288 time.sleep(0.01)
289
290 except ldap.SERVER_DOWN:
291 log.error(f"Delete worker {thread_id}: SERVER DOWN - ns-slapd crashed!")
292 mark_server_crashed()
293 stop_event.set()
294 break
295 except ldap.NO_SUCH_OBJECT:
296 pass
297 except Exception as e:
298 update_stats('errors')
299 if delete_count < 10:
300 log.debug(f"Delete worker {thread_id} error: {e}")
301 finally:
302 try:
303 conn.unbind_s()
304 except:
305 pass
306 log.info(f"Delete worker {thread_id} stopped (total deletes: {delete_count})")
307
308
309 def churn_worker(stop_event, host, port, thread_id):
310 """Worker thread for add/delete churn
311
312 :param stop_event: Event to signal thread termination
313 :param host: LDAP server hostname
314 :param port: LDAP server port
315 :param thread_id: Thread identifier
316 """
317 # Create independent raw LDAP connection for this thread
318 conn = create_raw_connection(host, port)
319 log.info(f"Churn worker {thread_id} started")
320 counter = 10000 + (thread_id * 10000)
321 add_count = 0
322
323 try:
324 while not stop_event.is_set():
325 try:
326 dn = f"uid=churn{counter},ou=people,{DEFAULT_SUFFIX}"
327
328 attrs = {
329 'objectClass': [b'top', b'person', b'inetOrgPerson'],
330 'uid': [f'churn{counter}'.encode()],
331 'cn': [f'Churn {counter}'.encode()],
332 'sn': [f'User{counter}'.encode()],
333 'description': [f'desc_{i}'.encode() for i in range(15)],
334 'telephoneNumber': [f'+1-555-churn-{i:04d}'.encode() for i in range(8)],
335 'userPassword': [b'password'],
336 }
337
338 conn.add_s(dn, modlist.addModlist(attrs))
339 add_count += 1
340 update_stats('adds')
341
342 time.sleep(0.005)
343
344 conn.delete_s(dn)
345 update_stats('deletes')
346
347 counter += 1
348
349 time.sleep(0.01)
350
351 except ldap.SERVER_DOWN:
352 log.error(f"Churn worker {thread_id}: SERVER DOWN - ns-slapd crashed!")
353 mark_server_crashed()
354 stop_event.set()
355 break
356 except ldap.ALREADY_EXISTS:
357 try:
358 conn.delete_s(dn)
359 except:
360 pass
361 counter += 1
362 except Exception as e:
363 update_stats('errors')
364 if add_count < 10:
365 log.debug(f"Churn worker {thread_id} error: {e}")
366 counter += 1
367 finally:
368 try:
369 conn.unbind_s()
370 except:
371 pass
372 log.info(f"Churn worker {thread_id} stopped (total adds: {add_count})")
373
374
375 def monitor_worker(stop_event, duration):
376 """Worker thread to monitor progress and enforce timeout
377
378 :param stop_event: Event to signal thread termination
379 :param duration: Test duration in seconds
380 """
381 log.info(f"Monitor worker started (duration: {duration}s / {duration/60:.1f}m)")
382
383 start_time = time.time()
384 last_stats_time = start_time
385
386 while not stop_event.is_set():
387 current_time = time.time()
388 elapsed = current_time - start_time
389
390 # Print stats every 30 seconds
391 if current_time - last_stats_time >= 30:
392 print_stats()
393 log.info(f"Progress: {elapsed:.0f}s / {duration}s ({elapsed/duration*100:.1f}%)")
394 last_stats_time = current_time
395
396 # Check if duration has been reached
397 if elapsed >= duration:
398 log.info(f"Duration {duration}s reached. Test completed successfully!")
399 stop_event.set()
400 break
401
402 time.sleep(1)
403
404 log.info("Monitor worker stopped")
405
406
407 @pytest.mark.slow
408 def test_bdb_cursor_race(topology_st):
409 """Stress test to reproduce BDB cursor race condition
410
411 :id: d53a9580-58e5-454c-96fd-57839660fd51
412 :setup: Standalone instance
413 :steps:
414 1. Setup test data with 1000 entries containing multi-valued attributes
415 2. Start search, modify, delete and churn threads
416 3. Run for 60 minutes
417 :expectedresults:
418 1. Success
419 2. Success
420 3. Server doesn't crash
421 """
422
423 inst = topology_st.standalone
424
425 log.info("=" * 80)
426 log.info("BDB Cursor Race Condition Test")
427 log.info(f"Instance: {inst.serverid}")
428 log.info(f"Base DN: {DEFAULT_SUFFIX}")
429 log.info(f"Duration: {DEFAULT_TIMEOUT}s ({DEFAULT_TIMEOUT/60:.0f} minutes)")
430 log.info("=" * 80)
431
432 # Enable log buffering for better performance
433 log.info("Enabling log buffering for access and error logs...")
434 inst.config.set('nsslapd-accesslog-logbuffering', 'on')
435 inst.config.set('nsslapd-errorlog-logbuffering', 'on')
436
437 # Setup test data
438 setup_test_data(inst, num_entries=1000)
439
440 # Extract connection parameters once to avoid sharing inst object
441 ldap_host = inst.host
442 ldap_port = inst.port
443
444 # Reset statistics and crash flag
445 global stats, server_crashed
446 stats = {
447 'searches': 0,
448 'modifies': 0,
449 'deletes': 0,
450 'adds': 0,
451 'errors': 0,
452 'start_time': time.time()
453 }
454 server_crashed = False
455
456 # Create stop event
457 stop_event = Event()
458
459 # Start all worker threads
460 threads = []
461
462 # Search workers - hammer equality indexes
463 for i in range(SEARCH_THREADS):
464 t = Thread(target=search_worker,
465 args=(stop_event, ldap_host, ldap_port, i),
466 name=f"search-{i}")
467 t.daemon = True
468 t.start()
469 threads.append(t)
470
471 # Modify workers - update multi-valued attributes
472 for i in range(MODIFY_THREADS):
473 t = Thread(target=modify_worker,
474 args=(stop_event, ldap_host, ldap_port, i),
475 name=f"modify-{i}")
476 t.daemon = True
477 t.start()
478 threads.append(t)
479
480 # Delete workers - trigger index cleanup
481 for i in range(DELETE_THREADS):
482 t = Thread(target=delete_worker,
483 args=(stop_event, ldap_host, ldap_port, i),
484 name=f"delete-{i}")
485 t.daemon = True
486 t.start()
487 threads.append(t)
488
489 # Churn workers - add/delete cycle
490 for i in range(CHURN_THREADS):
491 t = Thread(target=churn_worker,
492 args=(stop_event, ldap_host, ldap_port, i),
493 name=f"churn-{i}")
494 t.daemon = True
495 t.start()
496 threads.append(t)
497
498 # Monitor worker - enforce timeout and print stats
499 monitor_thread = Thread(target=monitor_worker,
500 args=(stop_event, DEFAULT_TIMEOUT),
501 name="monitor")
502 monitor_thread.daemon = True
503 monitor_thread.start()
504 threads.append(monitor_thread)
505
506 log.info("=" * 80)
507 log.info(f"Started {len(threads)} worker threads:")
508 log.info(f" - {SEARCH_THREADS} search threads (equality index queries)")
509 log.info(f" - {MODIFY_THREADS} modify threads (multi-valued attr updates)")
510 log.info(f" - {DELETE_THREADS} delete threads (index cleanup)")
511 log.info(f" - {CHURN_THREADS} churn threads (add/delete cycle)")
512 log.info(f" - 1 monitor thread (timeout enforcement)")
513 log.info("=" * 80)
514 log.info(f"Running for {DEFAULT_TIMEOUT}s ({DEFAULT_TIMEOUT/60:.0f} minutes)...")
515 log.info("=" * 80)
516
517 # Wait for all threads to finish or timeout
518 for t in threads:
519 t.join()
520
521 # Final statistics
522 log.info("=" * 80)
523 log.info("Final Statistics:")
524 print_stats()
525 log.info("=" * 80)
526
527 # Check if server crashed during the test
528 if server_crashed:
529 log.error("=" * 80)
530 log.error("Test FAILED: Server crashed during the test!")
531 log.error("=" * 80)
532 pytest.fail("ns-slapd crashed during BDB cursor race test")
533 else:
534 log.info("=" * 80)
535 log.info("Test PASSED: Completed full duration without crash")
536 log.info("=" * 80)
537
538
539 if __name__ == '__main__':
540 # Run isolated
541 # -s for DEBUG mode
542 import os
543 CURRENT_FILE = os.path.realpath(__file__)
544 pytest.main(["-s", "-v", CURRENT_FILE])