| 1 |
# --- BEGIN COPYRIGHT BLOCK --- |
| 2 |
# Copyright (C) 2026 Red Hat, Inc. |
| 3 |
# All rights reserved. |
| 4 |
# |
| 5 |
# License: GPL (version 3 or any later version). |
| 6 |
# See LICENSE for details. |
| 7 |
# --- END COPYRIGHT BLOCK --- |
| 8 |
# |
| 9 |
|
| 10 |
import pytest |
| 11 |
import os |
| 12 |
import random |
| 13 |
import re |
| 14 |
import string |
| 15 |
import subprocess |
| 16 |
import threading |
| 17 |
import time |
| 18 |
from contextlib import suppress, AbstractContextManager |
| 19 |
from lib389.backend import Backend, Backends |
| 20 |
from lib389.idm.user import UserAccounts |
| 21 |
from lib389.replica import Changelog, ReplicationManager, Replicas |
| 22 |
from lib389.utils import * |
| 23 |
from lib389._constants import * |
| 24 |
from lib389.cli_base import FakeArgs |
| 25 |
from lib389.cli_ctl.health import health_check_run |
| 26 |
from test389.topologies import topology_m2, topology_m3 |
| 27 |
from lib389.paths import Paths |
| 28 |
|
| 29 |
CMD_OUTPUT = 'No issues found.' |
| 30 |
JSON_OUTPUT = '[]' |
| 31 |
|
| 32 |
LOGIC_DICT = { |
| 33 |
False: ( "not ", "", lambda x: x ), |
| 34 |
True: ( "", "not ", lambda x: not x ) |
| 35 |
} |
| 36 |
|
| 37 |
ds_paths = Paths() |
| 38 |
log = logging.getLogger(__name__) |
| 39 |
|
| 40 |
|
| 41 |
class LoadInstance(AbstractContextManager): |
| 42 |
@staticmethod |
| 43 |
def create_test_user(inst): |
| 44 |
users = UserAccounts(inst, DEFAULT_SUFFIX) |
| 45 |
uid = str(20000 + int(inst.serverid[8:])) |
| 46 |
properties = { |
| 47 |
'uid': f'testuser_{inst.serverid}', |
| 48 |
'cn' : f'testuser_{inst.serverid}', |
| 49 |
'sn' : 'user_{inst.serverid}', |
| 50 |
'uidNumber' : uid, |
| 51 |
'gidNumber' : uid, |
| 52 |
'homeDirectory' : f'/home/testuser_{inst.serverid}' |
| 53 |
} |
| 54 |
return users.ensure_state(properties=properties) |
| 55 |
|
| 56 |
def __init__(self, inst): |
| 57 |
self.inst = inst |
| 58 |
self.stop = threading.Event() |
| 59 |
self.thread = threading.Thread(target=self.loader) |
| 60 |
self.user = LoadInstance.create_test_user(inst) |
| 61 |
|
| 62 |
def loader(self): |
| 63 |
while not self.stop.is_set(): |
| 64 |
value = ''.join(random.choices(string.ascii_uppercase + string.digits, k=10)) |
| 65 |
self.user.replace('description', value) |
| 66 |
#log.info(f'Modified {self.user.dn} description with {value} on {self.inst.serverid}') |
| 67 |
time.sleep(0.001) |
| 68 |
|
| 69 |
def __exit__(self, *args): |
| 70 |
self.stop.set() |
| 71 |
self.thread.join() |
| 72 |
self.user.delete() |
| 73 |
|
| 74 |
def __enter__(self): |
| 75 |
self.thread.start() |
| 76 |
return self |
| 77 |
|
| 78 |
|
| 79 |
class BreakReplication(AbstractContextManager): |
| 80 |
def __init__(self, topo, instances): |
| 81 |
self.topo = topo |
| 82 |
self.replicas = [ Replicas(inst).list()[0] for inst in instances ] |
| 83 |
self.oldvals = [] |
| 84 |
|
| 85 |
def __exit__(self, *args): |
| 86 |
for replica,oldval in self.oldvals: |
| 87 |
replica.replace('nsds5ReplicaBindDNGroup', oldval) |
| 88 |
|
| 89 |
def __enter__(self): |
| 90 |
# Ensure replication sessions are stopped to avoid race conditions |
| 91 |
self.topo.pause_all_replicas() |
| 92 |
for replica in self.replicas: |
| 93 |
oldval = replica.get_attr_val_utf8('nsds5ReplicaBindDNGroup') |
| 94 |
replica.replace('nsds5ReplicaBindDNGroup', 'cn=repl') |
| 95 |
self.oldvals.append((replica, oldval)) |
| 96 |
self.topo.resume_all_replicas() |
| 97 |
return self |
| 98 |
|
| 99 |
|
| 100 |
def assert_is_in_result(result, searched_code, isnot=False): |
| 101 |
# Assert if searched_code is not in logcap |
| 102 |
if searched_code is None: |
| 103 |
return |
| 104 |
|
| 105 |
# Handle positive and negative tests: |
| 106 |
nomatch, match, f = LOGIC_DICT[bool(isnot)] |
| 107 |
try: |
| 108 |
assert f(re.search(re.escape(searched_code), result)) |
| 109 |
log.info(f'Searched code {searched_code} is {match}in healthcheck output') |
| 110 |
except AssertionError as exc: |
| 111 |
log.error(f'{searched_code} is {nomatch}in healthcheck output: {result}') |
| 112 |
raise |
| 113 |
|
| 114 |
|
| 115 |
def run_healthcheck_and_check_result(topology, instance, searched_code, json, searched_code2=None, isnot=False): |
| 116 |
cmd = [ 'dsctl', ] |
| 117 |
if json: |
| 118 |
cmd.append('--json') |
| 119 |
if searched_code == CMD_OUTPUT: |
| 120 |
searched_code = JSON_OUTPUT |
| 121 |
cmd.append(instance.serverid) |
| 122 |
cmd.extend(['healthcheck', '--check', 'replication' , 'backends:userroot:cl_trimming']) |
| 123 |
|
| 124 |
result = subprocess.run(cmd, capture_output=True, universal_newlines=True) |
| 125 |
log.info(f'Running: {cmd}') |
| 126 |
log.info(f'Stdout: {result.stdout}') |
| 127 |
log.info(f'Stderr: {result.stdout}') |
| 128 |
log.info(f'Return code: {result.returncode}') |
| 129 |
stdout = result.stdout |
| 130 |
|
| 131 |
# stdout should not be empty |
| 132 |
assert stdout is not None |
| 133 |
assert len(stdout) > 0 |
| 134 |
assert_is_in_result(stdout, searched_code, isnot=isnot) |
| 135 |
assert_is_in_result(stdout, searched_code2, isnot=isnot) |
| 136 |
|
| 137 |
|
| 138 |
|
| 139 |
def set_changelog_trimming(instance): |
| 140 |
log.info('Get the changelog enteries') |
| 141 |
inst_changelog = Changelog(instance, suffix=DEFAULT_SUFFIX) |
| 142 |
|
| 143 |
log.info('Set nsslapd-changelogmaxage to 30d') |
| 144 |
inst_changelog.set_max_age('30d') |
| 145 |
|
| 146 |
|
| 147 |
@pytest.mark.xfail(ds_is_older("1.4.1"), reason="Not implemented") |
| 148 |
def test_healthcheck_replication_replica_not_reachable(topology_m2): |
| 149 |
"""Check if HealthCheck returns DSREPLLE0005 code |
| 150 |
|
| 151 |
:id: d452a564-7b82-4c1a-b331-a71abbd82a10 |
| 152 |
:setup: Replicated topology |
| 153 |
:steps: |
| 154 |
1. Create a replicated topology |
| 155 |
2. On M1, set nsds5replicaport for the replication agreement to an unreachable port on the replica |
| 156 |
3. Use HealthCheck without --json option |
| 157 |
4. Use HealthCheck with --json option |
| 158 |
5. On M1, set nsds5replicaport for the replication agreement to a reachable port number |
| 159 |
6. Use HealthCheck without --json option |
| 160 |
7. Use HealthCheck with --json option |
| 161 |
:expectedresults: |
| 162 |
1. Success |
| 163 |
2. Success |
| 164 |
3. Healthcheck reports DSREPLLE0005 code and related details |
| 165 |
4. Healthcheck reports DSREPLLE0005 code and related details |
| 166 |
5. Success |
| 167 |
6. Healthcheck reports no issue found |
| 168 |
7. Healthcheck reports no issue found |
| 169 |
""" |
| 170 |
|
| 171 |
RET_CODE = 'DSREPLLE0005' |
| 172 |
|
| 173 |
M1 = topology_m2.ms['supplier1'] |
| 174 |
M2 = topology_m2.ms['supplier2'] |
| 175 |
|
| 176 |
set_changelog_trimming(M1) |
| 177 |
|
| 178 |
log.info('Set nsds5replicaport for the replication agreement to an unreachable port') |
| 179 |
repl = ReplicationManager(DEFAULT_SUFFIX) |
| 180 |
repl.wait_for_replication(M1, M2) |
| 181 |
|
| 182 |
replica_m1 = Replicas(M1).get(DEFAULT_SUFFIX) |
| 183 |
agmt_m1 = replica_m1.get_agreements().list()[0] |
| 184 |
agmt_m1.replace('nsds5replicaport', '4389') |
| 185 |
# Should generates updates here to insure that we starts a new replication session |
| 186 |
# and really try to connect to the consumer |
| 187 |
with suppress(Exception): |
| 188 |
repl.wait_for_replication(M1, M2, timeout=5) |
| 189 |
|
| 190 |
run_healthcheck_and_check_result(topology_m2, M1, RET_CODE, json=False) |
| 191 |
run_healthcheck_and_check_result(topology_m2, M1, RET_CODE, json=True) |
| 192 |
|
| 193 |
log.info('Set nsds5replicaport for the replication agreement to a reachable port') |
| 194 |
agmt_m1.replace('nsDS5ReplicaPort', '{}'.format(M2.port)) |
| 195 |
repl.wait_for_replication(M1, M2) |
| 196 |
|
| 197 |
run_healthcheck_and_check_result(topology_m2, M1, CMD_OUTPUT, json=False) |
| 198 |
run_healthcheck_and_check_result(topology_m2, M1, JSON_OUTPUT, json=True) |
| 199 |
|
| 200 |
|
| 201 |
@pytest.mark.xfail(ds_is_older("1.4.1"), reason="Not implemented") |
| 202 |
def test_healthcheck_changelog_trimming_not_configured(topology_m2): |
| 203 |
"""Check if HealthCheck returns DSCLLE0001 code |
| 204 |
|
| 205 |
:id: c2165032-88ba-4978-a4ca-2fecfd8c35d8 |
| 206 |
:setup: Replicated topology |
| 207 |
:steps: |
| 208 |
1. Create a replicated topology |
| 209 |
2. On M1, check that value of nsslapd-changelogmaxage from cn=changelog5,cn=config is None |
| 210 |
3. Stop M1 and use HealthCheck without --json option (offline; reads dse.ldif) |
| 211 |
4. Stop M1 and use HealthCheck with --json option |
| 212 |
5. Start M1, set nsslapd-changelogmaxage to 30d |
| 213 |
6. Stop M1 and use HealthCheck without --json option |
| 214 |
7. Stop M1 and use HealthCheck with --json option |
| 215 |
:expectedresults: |
| 216 |
1. Success |
| 217 |
2. Success |
| 218 |
3. Healthcheck reports DSCLLE0001 code and related details |
| 219 |
4. Healthcheck reports DSCLLE0001 code and related details (json) |
| 220 |
5. Success |
| 221 |
6. Healthcheck reports no issue found |
| 222 |
7. Healthcheck reports no issue found (json) |
| 223 |
""" |
| 224 |
|
| 225 |
M1 = topology_m2.ms['supplier1'] |
| 226 |
|
| 227 |
RET_CODE = 'DSCLLE0001' |
| 228 |
|
| 229 |
log.info('Get the changelog entries for M1') |
| 230 |
changelog_m1 = Changelog(M1, suffix=DEFAULT_SUFFIX) |
| 231 |
|
| 232 |
log.info('Check nsslapd-changelogmaxage value') |
| 233 |
if changelog_m1.get_attr_val('nsslapd-changelogmaxage') is not None: |
| 234 |
changelog_m1.remove_all('nsslapd-changelogmaxage') |
| 235 |
|
| 236 |
time.sleep(3) |
| 237 |
|
| 238 |
try: |
| 239 |
log.info('Run healthcheck expect DSCLLE0001') |
| 240 |
|
| 241 |
run_healthcheck_and_check_result(topology_m2, M1, RET_CODE, json=False) |
| 242 |
run_healthcheck_and_check_result(topology_m2, M1, RET_CODE, json=True) |
| 243 |
|
| 244 |
log.info("Stopped server and testing offline mode ....") |
| 245 |
M1.stop() |
| 246 |
M1.lint_clear_dse_cache() |
| 247 |
run_healthcheck_and_check_result(topology_m2, M1, RET_CODE, json=False) |
| 248 |
run_healthcheck_and_check_result(topology_m2, M1, RET_CODE, json=True) |
| 249 |
|
| 250 |
log.info('Start M1 and configure changelog trimming (requires LDAP)') |
| 251 |
M1.start() |
| 252 |
set_changelog_trimming(M1) |
| 253 |
|
| 254 |
run_healthcheck_and_check_result(topology_m2, M1, CMD_OUTPUT, json=False) |
| 255 |
run_healthcheck_and_check_result(topology_m2, M1, JSON_OUTPUT, json=True) |
| 256 |
|
| 257 |
log.info("Stopped server and testing offline mode ....") |
| 258 |
M1.stop() |
| 259 |
M1.lint_clear_dse_cache() |
| 260 |
run_healthcheck_and_check_result(topology_m2, M1, CMD_OUTPUT, json=False) |
| 261 |
run_healthcheck_and_check_result(topology_m2, M1, JSON_OUTPUT, json=True) |
| 262 |
|
| 263 |
finally: |
| 264 |
if not M1.status(): |
| 265 |
M1.start() |
| 266 |
|
| 267 |
|
| 268 |
@pytest.mark.xfail(ds_is_older("1.4.1"), reason="Not implemented") |
| 269 |
def test_healthcheck_replication_presence_of_conflict_entries(topology_m2): |
| 270 |
"""Check if HealthCheck returns DSREPLLE0002 code |
| 271 |
|
| 272 |
:id: 43abc6c6-2075-42eb-8fa3-aa092ff64cba |
| 273 |
:setup: Replicated topology |
| 274 |
:steps: |
| 275 |
1. Create a replicated topology |
| 276 |
2. Create conflict entries : different entries renamed to the same dn |
| 277 |
3. Use HealthCheck without --json option |
| 278 |
4. Use HealthCheck with --json option |
| 279 |
:expectedresults: |
| 280 |
1. Success |
| 281 |
2. Success |
| 282 |
3. Healthcheck reports DSREPLLE0002 code and related details |
| 283 |
4. Healthcheck reports DSREPLLE0002 code and related details |
| 284 |
""" |
| 285 |
|
| 286 |
RET_CODE = 'DSREPLLE0002' |
| 287 |
|
| 288 |
M1 = topology_m2.ms['supplier1'] |
| 289 |
M2 = topology_m2.ms['supplier2'] |
| 290 |
|
| 291 |
repl = ReplicationManager(DEFAULT_SUFFIX) |
| 292 |
repl.wait_for_replication(M1, M2) |
| 293 |
|
| 294 |
topology_m2.pause_all_replicas() |
| 295 |
|
| 296 |
log.info("Create conflict entries") |
| 297 |
test_users_m1 = UserAccounts(M1, DEFAULT_SUFFIX) |
| 298 |
test_users_m2 = UserAccounts(M2, DEFAULT_SUFFIX) |
| 299 |
user_num = 1000 |
| 300 |
test_users_m1.create_test_user(user_num, 2000) |
| 301 |
test_users_m2.create_test_user(user_num, 2000) |
| 302 |
|
| 303 |
topology_m2.resume_all_replicas() |
| 304 |
|
| 305 |
repl.test_replication_topology(topology_m2) |
| 306 |
|
| 307 |
run_healthcheck_and_check_result(topology_m2, M1, RET_CODE, json=False) |
| 308 |
run_healthcheck_and_check_result(topology_m2, M1, RET_CODE, json=True) |
| 309 |
|
| 310 |
|
| 311 |
def test_healthcheck_non_replicated_suffixes(topology_m2): |
| 312 |
"""Check if backend lint function unexpectedly throws exception |
| 313 |
|
| 314 |
:id: f922edf8-c527-4802-9f42-0b75bf97098a |
| 315 |
:setup: 2 MMR topology |
| 316 |
:steps: |
| 317 |
1. Create a new suffix: cn=changelog |
| 318 |
2. Call healthcheck (there should not be any exceptions raised) |
| 319 |
:expectedresults: |
| 320 |
1. Success |
| 321 |
2. Success |
| 322 |
""" |
| 323 |
|
| 324 |
inst = topology_m2.ms['supplier1'] |
| 325 |
|
| 326 |
# Create second suffix |
| 327 |
backends = Backends(inst) |
| 328 |
backends.create(properties={'nsslapd-suffix': "cn=changelog", |
| 329 |
'name': 'changelog'}) |
| 330 |
|
| 331 |
# Call healthcheck |
| 332 |
args = FakeArgs() |
| 333 |
args.instance = inst.serverid |
| 334 |
args.verbose = inst.verbose |
| 335 |
args.list_errors = False |
| 336 |
args.list_checks = False |
| 337 |
args.check = ['backends'] |
| 338 |
args.dry_run = False |
| 339 |
args.json = False |
| 340 |
args.exclude_check = [] |
| 341 |
|
| 342 |
inst.stop() |
| 343 |
try: |
| 344 |
health_check_run(inst, topology_m2.logcap.log, args) |
| 345 |
finally: |
| 346 |
if not inst.status(): |
| 347 |
inst.start() |
| 348 |
health_check_run(inst, topology_m2.logcap.log, args) |
| 349 |
|
| 350 |
|
| 351 |
def test_healthcheck_replica_busy(topology_m3): |
| 352 |
"""Check that HealthCheck does not returns DSREPLLE0003 code when a replicva is busy |
| 353 |
|
| 354 |
:id: b7c4a5aa-ef98-11ef-87f5-482ae39447e5 |
| 355 |
:setup: 3 MMR topology |
| 356 |
:steps: |
| 357 |
1. Create a 3 suppliers full-mesh topology |
| 358 |
2. Generate constant modify load on S1 and S2 |
| 359 |
3. Wait a bit to ensure stable replication flow |
| 360 |
4. Perform a modify on S3 |
| 361 |
5. Use HealthCheck on S3 without --json option |
| 362 |
6. Use HealthCheck on S3 with --json option |
| 363 |
:expectedresults: |
| 364 |
1. Success |
| 365 |
2. Success |
| 366 |
3. Success |
| 367 |
4. Success |
| 368 |
5. Healthcheck should not reports DSREPLLE0003 code and related details |
| 369 |
6. Healthcheck should not reports DSREPLLE0003 code and related details |
| 370 |
""" |
| 371 |
|
| 372 |
RET_CODE = 'DSREPLLE0003' |
| 373 |
# Is DSREPLLE0003 ignored if replica is busy ? |
| 374 |
ignored = not ds_is_older("2.2") |
| 375 |
|
| 376 |
S1 = topology_m3.ms['supplier1'] |
| 377 |
S2 = topology_m3.ms['supplier2'] |
| 378 |
S3 = topology_m3.ms['supplier3'] |
| 379 |
with LoadInstance(S1), LoadInstance(S2): |
| 380 |
# Wait a bit to let replication starts |
| 381 |
time.sleep(10) |
| 382 |
# Create user on S3 then remove it: |
| 383 |
LoadInstance(S3).user.delete() |
| 384 |
# S3 agrements should now be in the replica busy state |
| 385 |
run_healthcheck_and_check_result(topology_m3, S3, RET_CODE, json=False, isnot=ignored) |
| 386 |
run_healthcheck_and_check_result(topology_m3, S3, RET_CODE, json=True, isnot=ignored) |
| 387 |
|
| 388 |
|
| 389 |
@pytest.mark.xfail(ds_is_older("1.4.1"), reason="Not implemented") |
| 390 |
def test_healthcheck_replication_out_of_sync_broken(topology_m3): |
| 391 |
"""Check if HealthCheck returns DSREPLLE0001 code |
| 392 |
|
| 393 |
:id: b5ae7cae-de0f-4206-95a4-f81538764bea |
| 394 |
:setup: 3 MMR topology |
| 395 |
:steps: |
| 396 |
1. Create a 3 suppliers full-mesh topology, on M2 and M3 don’t set nsds5BeginReplicaRefresh:start |
| 397 |
2. Perform modifications on M1 |
| 398 |
3. Use HealthCheck without --json option |
| 399 |
4. Use HealthCheck with --json option |
| 400 |
:expectedresults: |
| 401 |
1. Success |
| 402 |
2. Success |
| 403 |
3. Healthcheck reports DSREPLLE0001 code and related details |
| 404 |
4. Healthcheck reports DSREPLLE0001 code and related details |
| 405 |
""" |
| 406 |
|
| 407 |
RET_CODE = 'DSREPLLE0001' |
| 408 |
|
| 409 |
S1 = topology_m3.ms['supplier1'] |
| 410 |
S2 = topology_m3.ms['supplier2'] |
| 411 |
S3 = topology_m3.ms['supplier3'] |
| 412 |
|
| 413 |
log.info('Break supplier2 and supplier3') |
| 414 |
with BreakReplication(topology_m3, (S2, S3)): |
| 415 |
time.sleep(1) |
| 416 |
log.info('Perform update on supplier1') |
| 417 |
test_users_m1 = UserAccounts(S1, DEFAULT_SUFFIX) |
| 418 |
test_users_m1.create_test_user(1005, 2000) |
| 419 |
|
| 420 |
time.sleep(3) |
| 421 |
run_healthcheck_and_check_result(topology_m3, S1, RET_CODE, json=False) |
| 422 |
run_healthcheck_and_check_result(topology_m3, S1, RET_CODE, json=True) |
| 423 |
|
| 424 |
|
| 425 |
def test_healthcheck_replication_out_of_sync_not_broken(topology_m3): |
| 426 |
"""Check that HealthCheck returns no issues when replication is in progress |
| 427 |
|
| 428 |
:id: 8305000d-ba4d-4c00-8331-be0e8bd92150 |
| 429 |
:setup: 3 MMR topology |
| 430 |
:steps: |
| 431 |
1. Create a 3 suppliers full-mesh topology, all replicas being synchronized |
| 432 |
2. Generate constant load on two supplier |
| 433 |
3. Use HealthCheck without --json option |
| 434 |
4. Use HealthCheck with --json option |
| 435 |
:expectedresults: |
| 436 |
1. Success |
| 437 |
2. Success |
| 438 |
3. Healthcheck reports no issue found |
| 439 |
4. Healthcheck reports no issue found |
| 440 |
""" |
| 441 |
|
| 442 |
RET_CODE = CMD_OUTPUT |
| 443 |
|
| 444 |
S1 = topology_m3.ms['supplier1'] |
| 445 |
S2 = topology_m3.ms['supplier2'] |
| 446 |
S3 = topology_m3.ms['supplier3'] |
| 447 |
|
| 448 |
with LoadInstance(S1), LoadInstance(S2): |
| 449 |
# Wait a bit to let replication starts |
| 450 |
time.sleep(10) |
| 451 |
run_healthcheck_and_check_result(topology_m3, S1, RET_CODE, json=False) |
| 452 |
run_healthcheck_and_check_result(topology_m3, S1, RET_CODE, json=True) |
| 453 |
|
| 454 |
|
| 455 |
if __name__ == '__main__': |
| 456 |
# Run isolated |
| 457 |
# -s for DEBUG mode |
| 458 |
CURRENT_FILE = os.path.realpath(__file__) |