Debian Codegraph @

dirsrvtests/tests/suites/replication/repl_log_monitoring_test.py

raw
1 # --- BEGIN COPYRIGHT BLOCK ---
2 # Copyright (C) 2025 Red Hat, Inc.
3 # All rights reserved.
4 #
5 # License: GPL (version 3 or any later version).
6 # See LICENSE for details.
7 # --- END COPYRIGHT BLOCK ---
8 #
9 import os
10 import time
11 import shutil
12 import json
13 import pytest
14 import logging
15 import tempfile
16 from datetime import datetime, timezone, timedelta
17
18 from lib389.tasks import *
19 from lib389.utils import *
20 from lib389.backend import Backends
21 from test389.topologies import topology_m4 as topo_m4
22 from lib389.idm.user import UserAccount
23 from lib389.replica import ReplicationManager
24 from lib389.repltools import ReplicationLogAnalyzer, DSLogParser
25 from lib389._constants import *
26
27 pytestmark = pytest.mark.tier0
28
29 DEBUGGING = os.getenv("DEBUGGING", default=False)
30 if DEBUGGING:
31 logging.getLogger(__name__).setLevel(logging.DEBUG)
32 else:
33 logging.getLogger(__name__).setLevel(logging.INFO)
34 log = logging.getLogger(__name__)
35
36
37 def _generate_test_data(supplier, suffix, count, user_prefix="test_user"):
38 """Generate test users and modifications"""
39 test_users = []
40 for i in range(count):
41 user_dn = f'uid={user_prefix}_{i},{suffix}'
42 test_user = UserAccount(supplier, user_dn)
43 test_user.create(properties={
44 'uid': f'{user_prefix}_{i}',
45 'cn': f'Test User {i}',
46 'sn': f'User{i}',
47 'userPassword': 'password',
48 'uidNumber': str(1000 + i),
49 'gidNumber': '2000',
50 'homeDirectory': f'/home/{user_prefix}_{i}'
51 })
52
53 # Generate modifications
54 for j in range(3):
55 test_user.add('description', f'Description {j}')
56 test_user.replace('cn', f'Modified User {test_user.get_attr_val("uid")}')
57 for j in range(3):
58 try:
59 test_user.remove('description', f'Description {j}')
60 except Exception:
61 pass
62
63 test_users.append(test_user)
64
65 return test_users
66
67
68 def _cleanup_test_data(test_users, tmp_dir):
69 """Clean up test users and temporary directory"""
70 for user in test_users:
71 try:
72 if user.exists():
73 user.delete()
74 except Exception as e:
75 log.warning(f"Error cleaning up test user: {e}")
76
77 try:
78 shutil.rmtree(tmp_dir, ignore_errors=True)
79 except Exception as e:
80 log.error(f"Error cleaning up temporary directory: {e}")
81
82
83 def _cleanup_multi_suffix_test(test_users_by_suffix, tmp_dir, suppliers, extra_suffixes):
84 """Clean up multi-suffix test data"""
85 for users in test_users_by_suffix.values():
86 for user in users:
87 try:
88 if user.exists():
89 user.delete()
90 except Exception as e:
91 log.warning(f"Error cleaning up test user: {e}")
92
93 # Remove extra backends
94 for suffix in extra_suffixes:
95 for supplier in suppliers:
96 try:
97 backends = Backends(supplier)
98 backends.get(suffix).delete()
99 except Exception as e:
100 log.warning(f"Error removing backend for {suffix}: {e}")
101
102 try:
103 shutil.rmtree(tmp_dir, ignore_errors=True)
104 except Exception as e:
105 log.error(f"Error cleaning up temporary directory: {e}")
106
107
108 def _clear_access_logs(suppliers):
109 """Clear access logs for all suppliers and restart."""
110 for supplier in suppliers:
111 supplier.deleteAccessLogs(restart=True)
112
113
114 def _restart_suppliers(suppliers):
115 """Restart all suppliers."""
116 for supplier in suppliers:
117 supplier.restart()
118
119
120 def _get_log_dirs(suppliers):
121 """Return log directories for all suppliers."""
122 return [s.ds_paths.log_dir for s in suppliers]
123
124
125 def _load_json(path):
126 """Load and return JSON from file."""
127 with open(path, 'r') as f:
128 return json.load(f)
129
130
131 def _pause_agreements(supplier, suffix):
132 """Pause outbound agreements and return list of paused tuples."""
133 paused = []
134 for agmt in supplier.agreement.list(suffix=suffix):
135 supplier.agreement.pause(agmt.dn)
136 paused.append((supplier, agmt.dn))
137 return paused
138
139
140 def _resume_agreements(paused_agreements):
141 """Resume paused replication agreements."""
142 for supplier_obj, dn in paused_agreements:
143 try:
144 supplier_obj.agreement.resume(dn)
145 except Exception as e:
146 log.warning(f"Failed to resume agreement {dn}: {e}")
147
148
149 def _assert_csn_details_schema(json_data):
150 """Validate csnDetails presence and basic structure."""
151 assert 'csnDetails' in json_data, "Expected csnDetails in JSON output for drill-down"
152 csn_details = json_data['csnDetails']
153 if csn_details:
154 # Check structure of at least one CSN detail entry
155 first_csn = next(iter(csn_details.values()))
156 assert 'csn' in first_csn, "CSN detail should contain 'csn' field"
157 assert 'targetDn' in first_csn, "CSN detail should contain 'targetDn' field"
158 assert 'suffix' in first_csn, "CSN detail should contain 'suffix' field"
159 assert 'globalLag' in first_csn, "CSN detail should contain 'globalLag' field"
160 assert 'originServer' in first_csn, "CSN detail should contain 'originServer' field"
161 assert 'arrivals' in first_csn, "CSN detail should contain 'arrivals' list"
162 assert 'hops' in first_csn, "CSN detail should contain 'hops' list"
163 assert isinstance(first_csn['arrivals'], list), "arrivals should be a list"
164
165 # Verify arrivals structure
166 if first_csn['arrivals']:
167 first_arrival = first_csn['arrivals'][0]
168 assert 'server' in first_arrival, "Arrival should contain 'server' field"
169 assert 'timestamp' in first_arrival, "Arrival should contain 'timestamp' field"
170 assert 'relativeDelay' in first_arrival, "Arrival should contain 'relativeDelay' field"
171
172 # Verify csnId is included in datapoints for cross-reference
173 if 'replicationLags' in json_data and json_data['replicationLags'].get('series'):
174 for series in json_data['replicationLags']['series']:
175 for datapoint in series['datapoints']:
176 assert 'csnId' in datapoint, "Datapoint should contain 'csnId' for drill-down"
177
178 return csn_details
179
180
181 def _find_latest_logtime_for_prefix(log_dir, suffix, start_time, end_time, user_prefix):
182 latest = None
183 for fname in os.listdir(log_dir):
184 if not fname.startswith('access'):
185 continue
186 full_path = os.path.join(log_dir, fname)
187 parser = DSLogParser(
188 logname=full_path,
189 suffixes=[suffix],
190 tz=timezone.utc,
191 start_time=start_time,
192 end_time=end_time
193 )
194 for record in parser.parse_file():
195 target_dn = record.get('target_dn') or ''
196 if user_prefix in target_dn:
197 ts = record.get('timestamp')
198 if ts and (latest is None or ts > latest):
199 latest = ts
200 return latest
201
202
203 def test_replication_log_monitoring_basic(topo_m4):
204 """Test basic replication log monitoring functionality
205
206 :id: e62ed58b-1acd-4e7d-9cfd-948ded4cede8
207 :setup: Four suppliers replication setup
208 :steps:
209 1. Create test data with known replication patterns
210 2. Configure log monitoring with basic options
211 3. Generate and verify reports
212 4. Validate report contents
213 :expectedresults:
214 1. Test data should be properly replicated
215 2. Reports should be generated successfully
216 3. Report contents should match expected patterns
217 4. Reports should contain expected data and statistics
218 """
219 tmp_dir = tempfile.mkdtemp(prefix='repl_analysis_')
220 test_users = []
221 suppliers = [topo_m4.ms[f"supplier{i}"] for i in range(1, 5)]
222 paused_agreements = []
223
224 try:
225 # Clear logs and restart servers
226 _clear_access_logs(suppliers)
227
228 # Generate test data with known patterns
229 log.info('Creating test data...')
230 test_users = _generate_test_data(suppliers[0], DEFAULT_SUFFIX, 10)
231
232 # Wait for replication
233 repl = ReplicationManager(DEFAULT_SUFFIX)
234 repl.test_replication_topology(topo_m4)
235
236 # Restart to flush logs
237 _restart_suppliers(suppliers)
238
239 # Configure monitoring
240 log_dirs = _get_log_dirs(suppliers)
241 repl_monitor = ReplicationLogAnalyzer(
242 log_dirs=log_dirs,
243 suffixes=[DEFAULT_SUFFIX],
244 anonymous=False,
245 only_fully_replicated=True
246 )
247
248 # Parse logs and generate reports
249 repl_monitor.parse_logs()
250 generated_files = repl_monitor.generate_report(
251 output_dir=tmp_dir,
252 formats=['csv', 'json'],
253 report_name='basic_test'
254 )
255
256 # Verify report files exist and have content
257 for fmt in ['csv', 'json', 'summary']:
258 assert os.path.exists(generated_files[fmt])
259 assert os.path.getsize(generated_files[fmt]) > 0
260
261 # Verify CSV content
262 with open(generated_files['csv'], 'r') as f:
263 csv_content = f.read()
264 # Verify headers
265 assert 'Timestamp,Server,CSN,Suffix' in csv_content
266 # Verify all servers present
267 for supplier in suppliers:
268 assert supplier.serverid in csv_content
269 # Verify suffix
270 assert DEFAULT_SUFFIX in csv_content
271
272 # Verify PatternFly JSON content
273 json_data = _load_json(generated_files['json'])
274 assert 'replicationLags' in json_data
275 assert json_data['replicationLags']['series'], "Expected replication lag series in JSON output"
276
277 _assert_csn_details_schema(json_data)
278
279 # Verify JSON summary
280 summary = _load_json(generated_files['summary'])
281 assert 'analysis_summary' in summary
282 stats = summary['analysis_summary']
283
284 # Verify basic stats
285 assert stats['total_servers'] == len(suppliers)
286 assert stats['total_updates'] > 0
287 assert stats['updates_by_suffix'][DEFAULT_SUFFIX] > 0
288 assert 'average_lag' in stats
289 assert 'maximum_lag' in stats
290
291 finally:
292 _cleanup_test_data(test_users, tmp_dir)
293
294
295 def test_replication_log_monitoring_advanced(topo_m4):
296 """Test advanced replication monitoring features
297
298 :id: 5bb8fd9f-c3ed-4118-a2f9-fd5d733230c7
299 :setup: Four suppliers replication setup
300 :steps:
301 1. Test filtering options
302 2. Test time range filtering
303 3. Test anonymization
304 4. Verify lag calculations
305 :expectedresults:
306 1. Filtering should work as expected
307 2. Time range filtering should limit results
308 3. Anonymization should hide server names
309 4. Lag calculations should be accurate
310 """
311 tmp_dir = tempfile.mkdtemp(prefix='repl_analysis_')
312 test_users = []
313 suppliers = [topo_m4.ms[f"supplier{i}"] for i in range(1, 5)]
314
315 try:
316 # Clear logs and restart servers
317 _clear_access_logs(suppliers)
318
319 # Generate test data
320 start_time = datetime.now(timezone.utc)
321 test_users = _generate_test_data(suppliers[0], DEFAULT_SUFFIX, 20)
322
323 # Force some lag by delaying operations
324 time.sleep(2)
325 for user in test_users[10:]:
326 user.replace('description', 'Modified after delay')
327
328 # Wait for replication
329 repl = ReplicationManager(DEFAULT_SUFFIX)
330 repl.test_replication_topology(topo_m4)
331
332 end_time = datetime.now(timezone.utc)
333
334 # Restart to flush logs
335 _restart_suppliers(suppliers)
336
337 log_dirs = _get_log_dirs(suppliers)
338
339 # Test 1: Lag time filtering
340 repl_monitor = ReplicationLogAnalyzer(
341 log_dirs=log_dirs,
342 suffixes=[DEFAULT_SUFFIX],
343 lag_time_lowest=1.0
344 )
345 repl_monitor.parse_logs()
346 results1 = repl_monitor.build_result()
347
348 # Verify lag filtering:
349 # Only consider dict values, skip the special "__hop_lags__" (if present)
350 for csn, server_map in results1['lag'].items():
351 t_list = [
352 record['logtime']
353 for key, record in server_map.items()
354 if isinstance(record, dict) and key != '__hop_lags__'
355 ]
356 if not t_list:
357 # If no normal records exist, just skip
358 continue
359
360 lag_time = max(t_list) - min(t_list)
361 # Must be strictly > 1.0
362 assert lag_time > 1.0, f"Expected lag_time > 1.0, got {lag_time}"
363
364 # Test 2: Time range filtering
365 repl_monitor = ReplicationLogAnalyzer(
366 log_dirs=log_dirs,
367 suffixes=[DEFAULT_SUFFIX],
368 time_range={'start': start_time, 'end': end_time}
369 )
370 repl_monitor.parse_logs()
371 results2 = repl_monitor.build_result()
372
373 # Verify the 'start-time' in results is within or after our start_time
374 utc_start_time = datetime.fromtimestamp(results2['utc-start-time'], timezone.utc)
375 assert utc_start_time >= start_time, (
376 f"Expected start time >= {start_time}, got {utc_start_time}"
377 )
378 assert 'end-time' in results2
379 assert 'utc-end-time' in results2
380 utc_end_time = datetime.fromtimestamp(results2['utc-end-time'], timezone.utc)
381 assert utc_end_time >= utc_start_time, (
382 f"Expected end time >= start time, got {utc_end_time} < {utc_start_time}"
383 )
384 assert utc_end_time <= end_time + timedelta(seconds=5), (
385 f"Expected end time within requested range, got {utc_end_time} beyond {end_time}"
386 )
387
388 # Test 3: Anonymization
389 repl_monitor = ReplicationLogAnalyzer(
390 log_dirs=log_dirs,
391 suffixes=[DEFAULT_SUFFIX],
392 anonymous=True
393 )
394 repl_monitor.parse_logs()
395 generated_files = repl_monitor.generate_report(
396 output_dir=tmp_dir,
397 formats=['csv'],
398 report_name='anon_test'
399 )
400
401 # Verify anonymization
402 with open(generated_files['csv'], 'r') as f:
403 content = f.read()
404 for supplier in suppliers:
405 # Original supplier.serverid should NOT appear
406 assert supplier.serverid not in content, (
407 f"Found real server name {supplier.serverid} in CSV"
408 )
409 # Instead, placeholders like 'server_0' should exist
410 assert 'server_0' in content, "Expected 'server_0' placeholder not found in CSV"
411
412 finally:
413 _cleanup_test_data(test_users, tmp_dir)
414
415
416 def test_replication_log_monitoring_multi_suffix(topo_m4):
417 """Test multi-suffix replication monitoring
418
419 :id: 6ef38c42-4961-476f-9e72-488d99211b8b
420 :setup: Four suppliers replication setup
421 :steps:
422 1. Create multiple suffixes with different replication patterns
423 2. Generate reports for all suffixes
424 3. Verify suffix-specific statistics
425 :expectedresults:
426 1. All suffixes should be monitored
427 2. Reports should show correct per-suffix data
428 3. Statistics should be accurate for each suffix
429 """
430 tmp_dir = tempfile.mkdtemp(prefix='multi_suffix_repl_')
431 SUFFIX_2 = "dc=test2"
432 SUFFIX_3 = "dc=test3"
433 all_suffixes = [DEFAULT_SUFFIX, SUFFIX_2, SUFFIX_3]
434 test_users_by_suffix = {suffix: [] for suffix in all_suffixes}
435 suppliers = [topo_m4.ms[f"supplier{i}"] for i in range(1, 5)]
436
437 try:
438 # Setup additional suffixes
439 for suffix in [SUFFIX_2, SUFFIX_3]:
440 repl = ReplicationManager(suffix)
441 for supplier in suppliers:
442 props = {
443 'cn': f'userRoot_{suffix.split(",")[0][3:]}',
444 'nsslapd-suffix': suffix
445 }
446 backends = Backends(supplier)
447 be = backends.create(properties=props)
448 be.create_sample_entries('001004002')
449
450 if supplier == suppliers[0]:
451 repl.create_first_supplier(supplier)
452 else:
453 repl.join_supplier(suppliers[0], supplier)
454
455 # Create full mesh
456 for suffix in all_suffixes:
457 repl = ReplicationManager(suffix)
458 for i, s1 in enumerate(suppliers):
459 for s2 in suppliers[i+1:]:
460 repl.ensure_agreement(s1, s2)
461 repl.ensure_agreement(s2, s1)
462
463 # Allow initial topology to settle before capturing metrics
464 for suffix in all_suffixes:
465 repl = ReplicationManager(suffix)
466 repl.test_replication_topology(topo_m4)
467
468 _clear_access_logs(suppliers)
469
470 start_time = datetime.now(timezone.utc)
471
472 # Generate different amounts of test data per suffix
473 test_users_by_suffix[DEFAULT_SUFFIX] = _generate_test_data(
474 suppliers[0], DEFAULT_SUFFIX, 10
475 )
476 test_users_by_suffix[SUFFIX_2] = _generate_test_data(
477 suppliers[0], SUFFIX_2, 5, user_prefix="test2_user"
478 )
479 test_users_by_suffix[SUFFIX_3] = _generate_test_data(
480 suppliers[0], SUFFIX_3, 15, user_prefix="test3_user"
481 )
482
483 # Wait for replication of generated data
484 for suffix in all_suffixes:
485 repl = ReplicationManager(suffix)
486 repl.test_replication_topology(topo_m4)
487
488 # Give replication a moment to flush to logs before grabbing the end time
489 time.sleep(1)
490 end_time = datetime.now(timezone.utc)
491
492 # Restart to flush logs
493 _restart_suppliers(suppliers)
494
495 # Monitor all suffixes
496 log_dirs = _get_log_dirs(suppliers)
497 repl_monitor = ReplicationLogAnalyzer(
498 log_dirs=log_dirs,
499 suffixes=all_suffixes,
500 time_range={'start': start_time, 'end': end_time}
501 )
502
503 repl_monitor.parse_logs()
504 generated_files = repl_monitor.generate_report(
505 output_dir=tmp_dir,
506 formats=['csv', 'json'],
507 report_name='multi_suffix_test'
508 )
509
510 assert os.path.exists(generated_files['json'])
511
512 # Verify summary statistics
513 with open(generated_files['summary'], 'r') as f:
514 summary = json.load(f)
515 stats = summary['analysis_summary']
516
517 # Verify updates by suffix
518 updates = stats['updates_by_suffix']
519 assert len(updates) == len(all_suffixes)
520 for suffix in all_suffixes:
521 assert suffix in updates
522 assert updates[suffix] > 0
523
524 # Verify relative amounts
525 assert updates[SUFFIX_3] > updates[DEFAULT_SUFFIX]
526 assert updates[DEFAULT_SUFFIX] > updates[SUFFIX_2]
527
528 finally:
529 _cleanup_multi_suffix_test(
530 test_users_by_suffix,
531 tmp_dir,
532 suppliers,
533 [SUFFIX_2, SUFFIX_3]
534 )
535
536
537 def test_replication_log_monitoring_filter_combinations(topo_m4):
538 """Test complex combinations of filtering options and interactions
539
540 :id: 103fc0ac-f0b8-48f1-8cdf-1f6ff57f9672
541 :setup: Four suppliers replication setup
542 :steps:
543 1. Test multiple concurrent filters
544 2. Test filter interactions
545 3. Verify filter precedence
546 :expectedresults:
547 1. Multiple filters should work together correctly
548 2. Filter interactions should be predictable
549 3. Results should respect all applied filters
550 """
551 tmp_dir = tempfile.mkdtemp(prefix='repl_filter_test_')
552 test_users = []
553 suppliers = [topo_m4.ms[f"supplier{i}"] for i in range(1, 5)]
554 paused_agreements = []
555
556 try:
557 # Clear logs and restart servers
558 _clear_access_logs(suppliers)
559
560 # Generate varied test data
561 start_time = datetime.now(timezone.utc)
562 test_users = _generate_test_data(suppliers[0], DEFAULT_SUFFIX, 30)
563
564 # Create different lag patterns
565 # Pause outbound agreements from supplier1 to build a replication backlog
566 paused_agreements = _pause_agreements(suppliers[0], DEFAULT_SUFFIX)
567
568 for i, user in enumerate(test_users):
569 if i % 3 == 0:
570 time.sleep(1.0) # Short lag
571 elif i % 3 == 1:
572 time.sleep(2.0) # Medium lag
573 user.replace('description', f'Modified with lag pattern {i}')
574
575 time.sleep(3)
576
577 # Resume agreements one at a time to create staggered lag
578 for idx, (supplier_obj, dn) in enumerate(paused_agreements):
579 supplier_obj.agreement.resume(dn)
580 if idx < len(paused_agreements) - 1:
581 time.sleep(0.5)
582
583 paused_agreements.clear()
584
585 # Wait for replication
586 repl = ReplicationManager(DEFAULT_SUFFIX)
587 repl.test_replication_topology(topo_m4)
588 end_time = datetime.now(timezone.utc)
589
590 # Restart to flush logs
591 _restart_suppliers(suppliers)
592
593 log_dirs = _get_log_dirs(suppliers)
594
595 # Test combined filters
596 lag_threshold = 0.5
597 etime_threshold = 0.001
598 repl_monitor = ReplicationLogAnalyzer(
599 log_dirs=log_dirs,
600 suffixes=[DEFAULT_SUFFIX],
601 lag_time_lowest=lag_threshold,
602 etime_lowest=etime_threshold,
603 only_fully_replicated=True,
604 time_range={'start': start_time, 'end': end_time}
605 )
606
607 repl_monitor.parse_logs()
608 results = repl_monitor.build_result()
609
610 # Verify filter combinations
611 for csn, server_map in results['lag'].items():
612 t_list = [
613 record['logtime']
614 for key, record in server_map.items()
615 if isinstance(record, dict) and key != '__hop_lags__'
616 ]
617 if not t_list:
618 continue
619
620 lag_time = max(t_list) - min(t_list)
621
622 # Verify all filters were applied
623 assert lag_time >= lag_threshold, "Lag time filter not applied"
624 assert len(t_list) == len(suppliers), "Not fully replicated"
625
626 # Verify time range
627 for t in t_list:
628 dt = datetime.fromtimestamp(t, timezone.utc)
629 assert start_time <= dt <= end_time, "Time range filter violated"
630 finally:
631 _resume_agreements(paused_agreements)
632 _cleanup_test_data(test_users, tmp_dir)
633
634
635 def test_replication_log_monitoring_csn_details_edge_cases(topo_m4):
636 """Test CSN details edge cases and structure validation
637
638 :id: f43dc473-4428-4971-be4c-169c4a78726e
639 :setup: Four suppliers replication setup
640 :steps:
641 1. Test CSN details structure with various replication patterns
642 2. Verify arrivals ordering and hop lag calculations
643 3. Test partial replication scenarios
644 4. Verify origin server detection
645 :expectedresults:
646 1. CSN details should have correct structure
647 2. Arrivals should be ordered by timestamp
648 3. Partial replication should be detected
649 4. Origin server should be correctly identified
650 """
651 tmp_dir = tempfile.mkdtemp(prefix='repl_csn_edge_')
652 test_users = []
653 suppliers = [topo_m4.ms[f"supplier{i}"] for i in range(1, 5)]
654
655 try:
656 _clear_access_logs(suppliers)
657
658 log.info('Creating test data for CSN details edge case testing...')
659 test_users = _generate_test_data(suppliers[0], DEFAULT_SUFFIX, 5)
660
661 repl = ReplicationManager(DEFAULT_SUFFIX)
662 repl.test_replication_topology(topo_m4)
663
664 _restart_suppliers(suppliers)
665
666 log_dirs = _get_log_dirs(suppliers)
667 repl_monitor = ReplicationLogAnalyzer(
668 log_dirs=log_dirs,
669 suffixes=[DEFAULT_SUFFIX],
670 anonymous=False,
671 only_fully_replicated=True
672 )
673
674 repl_monitor.parse_logs()
675 generated_files = repl_monitor.generate_report(
676 output_dir=tmp_dir,
677 formats=['json'],
678 report_name='csn_edge_test'
679 )
680
681 assert os.path.exists(generated_files['json'])
682
683 json_data = _load_json(generated_files['json'])
684
685 assert 'csnDetails' in json_data, "csnDetails should be present"
686 csn_details = json_data['csnDetails']
687
688 if csn_details:
689 for csn, details in csn_details.items():
690 arrivals = details.get('arrivals', [])
691 if len(arrivals) > 1:
692 timestamps = [a['timestamp'] for a in arrivals]
693 assert timestamps == sorted(timestamps), \
694 f"Arrivals for CSN {csn} should be ordered by timestamp"
695
696 if arrivals:
697 origin_arrival = next((a for a in arrivals if a.get('isOrigin')), None)
698 assert origin_arrival is not None, "Expected an arrival marked as origin"
699 assert origin_arrival.get('server') == details.get('originServer'), \
700 "Origin arrival server should match originServer field"
701
702 for i, arrival in enumerate(arrivals[1:], start=1):
703 assert 'hopLag' in arrival, \
704 f"Arrival {i} should have hopLag"
705 assert arrival['hopLag'] >= 0, \
706 "hopLag should be non-negative"
707
708 if len(arrivals) > 1:
709 global_lag = details.get('globalLag', 0)
710 assert global_lag >= 0, "globalLag should be non-negative"
711
712 last_delay = arrivals[-1].get('relativeDelay', 0)
713 assert abs(last_delay - global_lag) < 0.001, \
714 "Last arrival's relativeDelay should match globalLag"
715
716 server_count = details.get('serverCount', 0)
717 assert server_count == len(arrivals), \
718 "serverCount should match number of arrivals"
719
720 total_hops = details.get('totalHops', 0)
721 expected_hops = max(0, len(arrivals) - 1)
722 assert total_hops == expected_hops, \
723 f"totalHops should be {expected_hops}, got {total_hops}"
724
725 hops = details.get('hops', [])
726 assert len(hops) == total_hops, \
727 "hops list length should match totalHops"
728
729 if 'replicationLags' in json_data and json_data['replicationLags'].get('series'):
730 for series in json_data['replicationLags']['series']:
731 for datapoint in series.get('datapoints', []):
732 csn_id = datapoint.get('csnId')
733 if csn_id:
734 assert csn_id in csn_details, \
735 f"csnId {csn_id} in datapoint should exist in csnDetails"
736
737 finally:
738 _cleanup_test_data(test_users, tmp_dir)
739
740
741 def test_replication_log_monitoring_origin_out_of_scope(topo_m4):
742 """Test origin detection when origin server record is outside time range
743
744 :id: d73fd9c5-f930-47d6-ade4-ada1cf0d2c21
745 :setup: Four suppliers replication setup
746 :steps:
747 1. Pause outbound agreements from the origin supplier
748 2. Generate changes, then resume agreements to delay consumer arrivals
749 3. Use time range that excludes the origin log but includes consumer logs
750 :expectedresults:
751 1. Origin server should be identified from replica ID mapping
752 2. At least one CSN should have origin outside the selected time range
753 3. JSON report should include csnDetails entries for validation
754 """
755 tmp_dir = tempfile.mkdtemp(prefix='repl_origin_scope_')
756 test_users = []
757 suppliers = [topo_m4.ms[f"supplier{i}"] for i in range(1, 5)]
758 paused_agreements = []
759
760 try:
761 # Reset access logs to make time-range cuts easier to reason about
762 _clear_access_logs(suppliers)
763
764 # Pause outbound agreements so consumer logs won't see the pre-resume CSNs
765 paused_agreements = _pause_agreements(suppliers[0], DEFAULT_SUFFIX)
766
767 log.info('Creating pre-resume changes for origin out-of-scope test...')
768 # Create CSNs that originate on supplier1 but won't reach consumers yet
769 pre_start = datetime.now(timezone.utc)
770 test_users = _generate_test_data(
771 suppliers[0], DEFAULT_SUFFIX, 2, user_prefix="origin_scope_pre"
772 )
773 pre_end = datetime.now(timezone.utc)
774 # Locate the latest origin log time for these CSNs to set a deterministic cutoff
775 origin_log_time = _find_latest_logtime_for_prefix(
776 suppliers[0].ds_paths.log_dir,
777 DEFAULT_SUFFIX,
778 pre_start,
779 pre_end,
780 "origin_scope_pre_"
781 )
782 assert origin_log_time is not None, "Expected origin server log entries for pre-resume data"
783 # Cut the analysis window just after origin logging, excluding supplier1 entries
784 start_time = origin_log_time + timedelta(seconds=1)
785 time.sleep(1)
786
787 # Resume agreements so the pre-resume CSNs replicate to consumers after start_time
788 _resume_agreements(paused_agreements)
789 paused_agreements.clear()
790
791 log.info('Creating post-resume changes for origin mapping...')
792 # Additional CSNs after resume ensure normal replication continues
793 test_users += _generate_test_data(
794 suppliers[0], DEFAULT_SUFFIX, 2, user_prefix="origin_scope_post"
795 )
796
797 # Wait for replication to finish and capture the upper bound of the time window
798 repl = ReplicationManager(DEFAULT_SUFFIX)
799 repl.test_replication_topology(topo_m4)
800 end_time = datetime.now(timezone.utc)
801
802 # Restart to flush logs before analysis
803 _restart_suppliers(suppliers)
804
805 log_dirs = _get_log_dirs(suppliers)
806 repl_monitor = ReplicationLogAnalyzer(
807 log_dirs=log_dirs,
808 suffixes=[DEFAULT_SUFFIX],
809 time_range={'start': start_time, 'end': end_time}
810 )
811
812 # Parse logs within the time window and produce JSON details
813 repl_monitor.parse_logs()
814 generated_files = repl_monitor.generate_report(
815 output_dir=tmp_dir,
816 formats=['json'],
817 report_name='origin_scope_test'
818 )
819
820 json_data = _load_json(generated_files['json'])
821
822 csn_details = json_data.get('csnDetails', {})
823 origin_server = suppliers[0].serverid
824 found = False
825 if csn_details:
826 origin_counts = {}
827 for details in csn_details.values():
828 origin = details.get('originServer', 'unknown')
829 origin_counts[origin] = origin_counts.get(origin, 0) + 1
830 if origin_server not in origin_counts and f"slapd-{origin_server}" in origin_counts:
831 origin_server = f"slapd-{origin_server}"
832
833 # Focus on the pre-resume CSNs; these should have origin out of scope
834 pre_details = [
835 details for details in csn_details.values()
836 if "origin_scope_pre_" in (details.get('targetDn') or '')
837 ]
838 assert pre_details, "Expected pre-resume CSNs in csnDetails"
839 # Confirm at least one CSN shows origin server missing from arrivals
840 for details in pre_details:
841 if details.get('originServer') != origin_server:
842 continue
843 arrivals = details.get('arrivals', [])
844 arrival_servers = {a.get('server') for a in arrivals}
845 if arrival_servers and origin_server not in arrival_servers:
846 found = True
847 break
848 log.info(
849 "Origin out-of-scope candidate: csn=%s arrivals=%s",
850 details.get('csn'),
851 sorted(arrival_servers)
852 )
853
854 assert found, (
855 "Expected at least one CSN where the origin server is outside the time range "
856 "but still identified via replica ID mapping"
857 )
858
859 finally:
860 _resume_agreements(paused_agreements)
861 _cleanup_test_data(test_users, tmp_dir)
862
863
864 def test_replication_log_monitoring_partial_replication(topo_m4):
865 """Test CSN details with partial replication (not all servers reached)
866
867 :id: d4026fd0-d83b-400e-8c2e-44fcf676368f
868 :setup: Four suppliers replication setup
869 :steps:
870 1. Pause replication agreements to create partial replication
871 2. Generate changes and verify partial replication detection
872 3. Verify replicatedToAll flag is correct
873 :expectedresults:
874 1. Partial replication should be detected
875 2. replicatedToAll should be False for partially replicated CSNs
876 3. serverCount should reflect actual servers reached
877 """
878 tmp_dir = tempfile.mkdtemp(prefix='repl_partial_')
879 test_users = []
880 suppliers = [topo_m4.ms[f"supplier{i}"] for i in range(1, 5)]
881 paused_agreements = []
882
883 try:
884 _clear_access_logs(suppliers)
885
886 log.info('Creating fully replicated test data...')
887 test_users = _generate_test_data(suppliers[0], DEFAULT_SUFFIX, 3)
888
889 repl = ReplicationManager(DEFAULT_SUFFIX)
890 repl.test_replication_topology(topo_m4)
891
892 _restart_suppliers(suppliers)
893
894 # Pause outbound agreements from supplier1 to create partial replication
895 paused_agreements = _pause_agreements(suppliers[0], DEFAULT_SUFFIX)
896
897 log.info('Creating partially replicated test data...')
898 test_users += _generate_test_data(suppliers[0], DEFAULT_SUFFIX, 3, user_prefix="partial_user")
899
900 # Allow some time for local logging; do not wait for full replication
901 time.sleep(2)
902
903 log_dirs = _get_log_dirs(suppliers)
904 repl_monitor = ReplicationLogAnalyzer(
905 log_dirs=log_dirs,
906 suffixes=[DEFAULT_SUFFIX],
907 anonymous=False,
908 only_fully_replicated=False,
909 only_not_replicated=False
910 )
911
912 repl_monitor.parse_logs()
913 generated_files = repl_monitor.generate_report(
914 output_dir=tmp_dir,
915 formats=['json'],
916 report_name='partial_repl_test'
917 )
918
919 json_data = _load_json(generated_files['json'])
920
921 assert 'csnDetails' in json_data
922 csn_details = json_data['csnDetails']
923
924 if csn_details:
925 fully_replicated_count = sum(
926 1 for details in csn_details.values()
927 if details.get('replicatedToAll', False)
928 )
929
930 assert fully_replicated_count > 0, \
931 "Should have some fully replicated CSNs"
932
933 total_servers = len(suppliers)
934 for csn, details in csn_details.items():
935 server_count = details.get('serverCount', 0)
936 assert server_count <= total_servers, \
937 f"serverCount ({server_count}) should not exceed total servers ({total_servers})"
938
939 replicated_to_all = details.get('replicatedToAll', False)
940 if replicated_to_all:
941 assert server_count == total_servers, \
942 "replicatedToAll=True requires serverCount == total_servers"
943
944 finally:
945 _resume_agreements(paused_agreements)
946 _cleanup_test_data(test_users, tmp_dir)
947
948
949 if __name__ == '__main__':
950 # Run isolated
951 # -s for DEBUG mode
952 CURRENT_FILE = os.path.realpath(__file__)
953 pytest.main("-s %s" % CURRENT_FILE)