Debian Codegraph @

dirsrvtests/tests/suites/healthcheck/health_repl_test.py

raw
1 # --- BEGIN COPYRIGHT BLOCK ---
2 # Copyright (C) 2026 Red Hat, Inc.
3 # All rights reserved.
4 #
5 # License: GPL (version 3 or any later version).
6 # See LICENSE for details.
7 # --- END COPYRIGHT BLOCK ---
8 #
9
10 import pytest
11 import os
12 import random
13 import re
14 import string
15 import subprocess
16 import threading
17 import time
18 from contextlib import suppress, AbstractContextManager
19 from lib389.backend import Backend, Backends
20 from lib389.idm.user import UserAccounts
21 from lib389.replica import Changelog, ReplicationManager, Replicas
22 from lib389.utils import *
23 from lib389._constants import *
24 from lib389.cli_base import FakeArgs
25 from lib389.cli_ctl.health import health_check_run
26 from test389.topologies import topology_m2, topology_m3
27 from lib389.paths import Paths
28
29 CMD_OUTPUT = 'No issues found.'
30 JSON_OUTPUT = '[]'
31
32 LOGIC_DICT = {
33 False: ( "not ", "", lambda x: x ),
34 True: ( "", "not ", lambda x: not x )
35 }
36
37 ds_paths = Paths()
38 log = logging.getLogger(__name__)
39
40
41 class LoadInstance(AbstractContextManager):
42 @staticmethod
43 def create_test_user(inst):
44 users = UserAccounts(inst, DEFAULT_SUFFIX)
45 uid = str(20000 + int(inst.serverid[8:]))
46 properties = {
47 'uid': f'testuser_{inst.serverid}',
48 'cn' : f'testuser_{inst.serverid}',
49 'sn' : 'user_{inst.serverid}',
50 'uidNumber' : uid,
51 'gidNumber' : uid,
52 'homeDirectory' : f'/home/testuser_{inst.serverid}'
53 }
54 return users.ensure_state(properties=properties)
55
56 def __init__(self, inst):
57 self.inst = inst
58 self.stop = threading.Event()
59 self.thread = threading.Thread(target=self.loader)
60 self.user = LoadInstance.create_test_user(inst)
61
62 def loader(self):
63 while not self.stop.is_set():
64 value = ''.join(random.choices(string.ascii_uppercase + string.digits, k=10))
65 self.user.replace('description', value)
66 #log.info(f'Modified {self.user.dn} description with {value} on {self.inst.serverid}')
67 time.sleep(0.001)
68
69 def __exit__(self, *args):
70 self.stop.set()
71 self.thread.join()
72 self.user.delete()
73
74 def __enter__(self):
75 self.thread.start()
76 return self
77
78
79 class BreakReplication(AbstractContextManager):
80 def __init__(self, topo, instances):
81 self.topo = topo
82 self.replicas = [ Replicas(inst).list()[0] for inst in instances ]
83 self.oldvals = []
84
85 def __exit__(self, *args):
86 for replica,oldval in self.oldvals:
87 replica.replace('nsds5ReplicaBindDNGroup', oldval)
88
89 def __enter__(self):
90 # Ensure replication sessions are stopped to avoid race conditions
91 self.topo.pause_all_replicas()
92 for replica in self.replicas:
93 oldval = replica.get_attr_val_utf8('nsds5ReplicaBindDNGroup')
94 replica.replace('nsds5ReplicaBindDNGroup', 'cn=repl')
95 self.oldvals.append((replica, oldval))
96 self.topo.resume_all_replicas()
97 return self
98
99
100 def assert_is_in_result(result, searched_code, isnot=False):
101 # Assert if searched_code is not in logcap
102 if searched_code is None:
103 return
104
105 # Handle positive and negative tests:
106 nomatch, match, f = LOGIC_DICT[bool(isnot)]
107 try:
108 assert f(re.search(re.escape(searched_code), result))
109 log.info(f'Searched code {searched_code} is {match}in healthcheck output')
110 except AssertionError as exc:
111 log.error(f'{searched_code} is {nomatch}in healthcheck output: {result}')
112 raise
113
114
115 def run_healthcheck_and_check_result(topology, instance, searched_code, json, searched_code2=None, isnot=False):
116 cmd = [ 'dsctl', ]
117 if json:
118 cmd.append('--json')
119 if searched_code == CMD_OUTPUT:
120 searched_code = JSON_OUTPUT
121 cmd.append(instance.serverid)
122 cmd.extend(['healthcheck', '--check', 'replication' , 'backends:userroot:cl_trimming'])
123
124 result = subprocess.run(cmd, capture_output=True, universal_newlines=True)
125 log.info(f'Running: {cmd}')
126 log.info(f'Stdout: {result.stdout}')
127 log.info(f'Stderr: {result.stdout}')
128 log.info(f'Return code: {result.returncode}')
129 stdout = result.stdout
130
131 # stdout should not be empty
132 assert stdout is not None
133 assert len(stdout) > 0
134 assert_is_in_result(stdout, searched_code, isnot=isnot)
135 assert_is_in_result(stdout, searched_code2, isnot=isnot)
136
137
138
139 def set_changelog_trimming(instance):
140 log.info('Get the changelog enteries')
141 inst_changelog = Changelog(instance, suffix=DEFAULT_SUFFIX)
142
143 log.info('Set nsslapd-changelogmaxage to 30d')
144 inst_changelog.set_max_age('30d')
145
146
147 @pytest.mark.xfail(ds_is_older("1.4.1"), reason="Not implemented")
148 def test_healthcheck_replication_replica_not_reachable(topology_m2):
149 """Check if HealthCheck returns DSREPLLE0005 code
150
151 :id: d452a564-7b82-4c1a-b331-a71abbd82a10
152 :setup: Replicated topology
153 :steps:
154 1. Create a replicated topology
155 2. On M1, set nsds5replicaport for the replication agreement to an unreachable port on the replica
156 3. Use HealthCheck without --json option
157 4. Use HealthCheck with --json option
158 5. On M1, set nsds5replicaport for the replication agreement to a reachable port number
159 6. Use HealthCheck without --json option
160 7. Use HealthCheck with --json option
161 :expectedresults:
162 1. Success
163 2. Success
164 3. Healthcheck reports DSREPLLE0005 code and related details
165 4. Healthcheck reports DSREPLLE0005 code and related details
166 5. Success
167 6. Healthcheck reports no issue found
168 7. Healthcheck reports no issue found
169 """
170
171 RET_CODE = 'DSREPLLE0005'
172
173 M1 = topology_m2.ms['supplier1']
174 M2 = topology_m2.ms['supplier2']
175
176 set_changelog_trimming(M1)
177
178 log.info('Set nsds5replicaport for the replication agreement to an unreachable port')
179 repl = ReplicationManager(DEFAULT_SUFFIX)
180 repl.wait_for_replication(M1, M2)
181
182 replica_m1 = Replicas(M1).get(DEFAULT_SUFFIX)
183 agmt_m1 = replica_m1.get_agreements().list()[0]
184 agmt_m1.replace('nsds5replicaport', '4389')
185 # Should generates updates here to insure that we starts a new replication session
186 # and really try to connect to the consumer
187 with suppress(Exception):
188 repl.wait_for_replication(M1, M2, timeout=5)
189
190 run_healthcheck_and_check_result(topology_m2, M1, RET_CODE, json=False)
191 run_healthcheck_and_check_result(topology_m2, M1, RET_CODE, json=True)
192
193 log.info('Set nsds5replicaport for the replication agreement to a reachable port')
194 agmt_m1.replace('nsDS5ReplicaPort', '{}'.format(M2.port))
195 repl.wait_for_replication(M1, M2)
196
197 run_healthcheck_and_check_result(topology_m2, M1, CMD_OUTPUT, json=False)
198 run_healthcheck_and_check_result(topology_m2, M1, JSON_OUTPUT, json=True)
199
200
201 @pytest.mark.xfail(ds_is_older("1.4.1"), reason="Not implemented")
202 def test_healthcheck_changelog_trimming_not_configured(topology_m2):
203 """Check if HealthCheck returns DSCLLE0001 code
204
205 :id: c2165032-88ba-4978-a4ca-2fecfd8c35d8
206 :setup: Replicated topology
207 :steps:
208 1. Create a replicated topology
209 2. On M1, check that value of nsslapd-changelogmaxage from cn=changelog5,cn=config is None
210 3. Stop M1 and use HealthCheck without --json option (offline; reads dse.ldif)
211 4. Stop M1 and use HealthCheck with --json option
212 5. Start M1, set nsslapd-changelogmaxage to 30d
213 6. Stop M1 and use HealthCheck without --json option
214 7. Stop M1 and use HealthCheck with --json option
215 :expectedresults:
216 1. Success
217 2. Success
218 3. Healthcheck reports DSCLLE0001 code and related details
219 4. Healthcheck reports DSCLLE0001 code and related details (json)
220 5. Success
221 6. Healthcheck reports no issue found
222 7. Healthcheck reports no issue found (json)
223 """
224
225 M1 = topology_m2.ms['supplier1']
226
227 RET_CODE = 'DSCLLE0001'
228
229 log.info('Get the changelog entries for M1')
230 changelog_m1 = Changelog(M1, suffix=DEFAULT_SUFFIX)
231
232 log.info('Check nsslapd-changelogmaxage value')
233 if changelog_m1.get_attr_val('nsslapd-changelogmaxage') is not None:
234 changelog_m1.remove_all('nsslapd-changelogmaxage')
235
236 time.sleep(3)
237
238 try:
239 log.info('Run healthcheck expect DSCLLE0001')
240
241 run_healthcheck_and_check_result(topology_m2, M1, RET_CODE, json=False)
242 run_healthcheck_and_check_result(topology_m2, M1, RET_CODE, json=True)
243
244 log.info("Stopped server and testing offline mode ....")
245 M1.stop()
246 M1.lint_clear_dse_cache()
247 run_healthcheck_and_check_result(topology_m2, M1, RET_CODE, json=False)
248 run_healthcheck_and_check_result(topology_m2, M1, RET_CODE, json=True)
249
250 log.info('Start M1 and configure changelog trimming (requires LDAP)')
251 M1.start()
252 set_changelog_trimming(M1)
253
254 run_healthcheck_and_check_result(topology_m2, M1, CMD_OUTPUT, json=False)
255 run_healthcheck_and_check_result(topology_m2, M1, JSON_OUTPUT, json=True)
256
257 log.info("Stopped server and testing offline mode ....")
258 M1.stop()
259 M1.lint_clear_dse_cache()
260 run_healthcheck_and_check_result(topology_m2, M1, CMD_OUTPUT, json=False)
261 run_healthcheck_and_check_result(topology_m2, M1, JSON_OUTPUT, json=True)
262
263 finally:
264 if not M1.status():
265 M1.start()
266
267
268 @pytest.mark.xfail(ds_is_older("1.4.1"), reason="Not implemented")
269 def test_healthcheck_replication_presence_of_conflict_entries(topology_m2):
270 """Check if HealthCheck returns DSREPLLE0002 code
271
272 :id: 43abc6c6-2075-42eb-8fa3-aa092ff64cba
273 :setup: Replicated topology
274 :steps:
275 1. Create a replicated topology
276 2. Create conflict entries : different entries renamed to the same dn
277 3. Use HealthCheck without --json option
278 4. Use HealthCheck with --json option
279 :expectedresults:
280 1. Success
281 2. Success
282 3. Healthcheck reports DSREPLLE0002 code and related details
283 4. Healthcheck reports DSREPLLE0002 code and related details
284 """
285
286 RET_CODE = 'DSREPLLE0002'
287
288 M1 = topology_m2.ms['supplier1']
289 M2 = topology_m2.ms['supplier2']
290
291 repl = ReplicationManager(DEFAULT_SUFFIX)
292 repl.wait_for_replication(M1, M2)
293
294 topology_m2.pause_all_replicas()
295
296 log.info("Create conflict entries")
297 test_users_m1 = UserAccounts(M1, DEFAULT_SUFFIX)
298 test_users_m2 = UserAccounts(M2, DEFAULT_SUFFIX)
299 user_num = 1000
300 test_users_m1.create_test_user(user_num, 2000)
301 test_users_m2.create_test_user(user_num, 2000)
302
303 topology_m2.resume_all_replicas()
304
305 repl.test_replication_topology(topology_m2)
306
307 run_healthcheck_and_check_result(topology_m2, M1, RET_CODE, json=False)
308 run_healthcheck_and_check_result(topology_m2, M1, RET_CODE, json=True)
309
310
311 def test_healthcheck_non_replicated_suffixes(topology_m2):
312 """Check if backend lint function unexpectedly throws exception
313
314 :id: f922edf8-c527-4802-9f42-0b75bf97098a
315 :setup: 2 MMR topology
316 :steps:
317 1. Create a new suffix: cn=changelog
318 2. Call healthcheck (there should not be any exceptions raised)
319 :expectedresults:
320 1. Success
321 2. Success
322 """
323
324 inst = topology_m2.ms['supplier1']
325
326 # Create second suffix
327 backends = Backends(inst)
328 backends.create(properties={'nsslapd-suffix': "cn=changelog",
329 'name': 'changelog'})
330
331 # Call healthcheck
332 args = FakeArgs()
333 args.instance = inst.serverid
334 args.verbose = inst.verbose
335 args.list_errors = False
336 args.list_checks = False
337 args.check = ['backends']
338 args.dry_run = False
339 args.json = False
340 args.exclude_check = []
341
342 inst.stop()
343 try:
344 health_check_run(inst, topology_m2.logcap.log, args)
345 finally:
346 if not inst.status():
347 inst.start()
348 health_check_run(inst, topology_m2.logcap.log, args)
349
350
351 def test_healthcheck_replica_busy(topology_m3):
352 """Check that HealthCheck does not returns DSREPLLE0003 code when a replicva is busy
353
354 :id: b7c4a5aa-ef98-11ef-87f5-482ae39447e5
355 :setup: 3 MMR topology
356 :steps:
357 1. Create a 3 suppliers full-mesh topology
358 2. Generate constant modify load on S1 and S2
359 3. Wait a bit to ensure stable replication flow
360 4. Perform a modify on S3
361 5. Use HealthCheck on S3 without --json option
362 6. Use HealthCheck on S3 with --json option
363 :expectedresults:
364 1. Success
365 2. Success
366 3. Success
367 4. Success
368 5. Healthcheck should not reports DSREPLLE0003 code and related details
369 6. Healthcheck should not reports DSREPLLE0003 code and related details
370 """
371
372 RET_CODE = 'DSREPLLE0003'
373 # Is DSREPLLE0003 ignored if replica is busy ?
374 ignored = not ds_is_older("2.2")
375
376 S1 = topology_m3.ms['supplier1']
377 S2 = topology_m3.ms['supplier2']
378 S3 = topology_m3.ms['supplier3']
379 with LoadInstance(S1), LoadInstance(S2):
380 # Wait a bit to let replication starts
381 time.sleep(10)
382 # Create user on S3 then remove it:
383 LoadInstance(S3).user.delete()
384 # S3 agrements should now be in the replica busy state
385 run_healthcheck_and_check_result(topology_m3, S3, RET_CODE, json=False, isnot=ignored)
386 run_healthcheck_and_check_result(topology_m3, S3, RET_CODE, json=True, isnot=ignored)
387
388
389 @pytest.mark.xfail(ds_is_older("1.4.1"), reason="Not implemented")
390 def test_healthcheck_replication_out_of_sync_broken(topology_m3):
391 """Check if HealthCheck returns DSREPLLE0001 code
392
393 :id: b5ae7cae-de0f-4206-95a4-f81538764bea
394 :setup: 3 MMR topology
395 :steps:
396 1. Create a 3 suppliers full-mesh topology, on M2 and M3 don’t set nsds5BeginReplicaRefresh:start
397 2. Perform modifications on M1
398 3. Use HealthCheck without --json option
399 4. Use HealthCheck with --json option
400 :expectedresults:
401 1. Success
402 2. Success
403 3. Healthcheck reports DSREPLLE0001 code and related details
404 4. Healthcheck reports DSREPLLE0001 code and related details
405 """
406
407 RET_CODE = 'DSREPLLE0001'
408
409 S1 = topology_m3.ms['supplier1']
410 S2 = topology_m3.ms['supplier2']
411 S3 = topology_m3.ms['supplier3']
412
413 log.info('Break supplier2 and supplier3')
414 with BreakReplication(topology_m3, (S2, S3)):
415 time.sleep(1)
416 log.info('Perform update on supplier1')
417 test_users_m1 = UserAccounts(S1, DEFAULT_SUFFIX)
418 test_users_m1.create_test_user(1005, 2000)
419
420 time.sleep(3)
421 run_healthcheck_and_check_result(topology_m3, S1, RET_CODE, json=False)
422 run_healthcheck_and_check_result(topology_m3, S1, RET_CODE, json=True)
423
424
425 def test_healthcheck_replication_out_of_sync_not_broken(topology_m3):
426 """Check that HealthCheck returns no issues when replication is in progress
427
428 :id: 8305000d-ba4d-4c00-8331-be0e8bd92150
429 :setup: 3 MMR topology
430 :steps:
431 1. Create a 3 suppliers full-mesh topology, all replicas being synchronized
432 2. Generate constant load on two supplier
433 3. Use HealthCheck without --json option
434 4. Use HealthCheck with --json option
435 :expectedresults:
436 1. Success
437 2. Success
438 3. Healthcheck reports no issue found
439 4. Healthcheck reports no issue found
440 """
441
442 RET_CODE = CMD_OUTPUT
443
444 S1 = topology_m3.ms['supplier1']
445 S2 = topology_m3.ms['supplier2']
446 S3 = topology_m3.ms['supplier3']
447
448 with LoadInstance(S1), LoadInstance(S2):
449 # Wait a bit to let replication starts
450 time.sleep(10)
451 run_healthcheck_and_check_result(topology_m3, S1, RET_CODE, json=False)
452 run_healthcheck_and_check_result(topology_m3, S1, RET_CODE, json=True)
453
454
455 if __name__ == '__main__':
456 # Run isolated
457 # -s for DEBUG mode
458 CURRENT_FILE = os.path.realpath(__file__)