| 1 |
# --- BEGIN COPYRIGHT BLOCK --- |
| 2 |
# Copyright (C) 2025 Red Hat, Inc. |
| 3 |
# All rights reserved. |
| 4 |
# |
| 5 |
# License: GPL (version 3 or any later version). |
| 6 |
# See LICENSE for details. |
| 7 |
# --- END COPYRIGHT BLOCK --- |
| 8 |
# |
| 9 |
import logging |
| 10 |
import pytest |
| 11 |
import os |
| 12 |
import re |
| 13 |
import time |
| 14 |
from lib389._constants import DEFAULT_SUFFIX, SUFFIX, LOG_REPLICA, LOG_DEFAULT |
| 15 |
from test389.topologies import topology_m4 |
| 16 |
from lib389.replica import Replicas, ReplicationManager |
| 17 |
from lib389.idm.user import UserAccounts, UserAccount |
| 18 |
from lib389.properties import TASK_WAIT |
| 19 |
|
| 20 |
pytestmark = pytest.mark.tier2 |
| 21 |
|
| 22 |
USER_CN = "test_user" |
| 23 |
log = logging.getLogger(__name__) |
| 24 |
|
| 25 |
|
| 26 |
@pytest.fixture(scope="function") |
| 27 |
def setup_replication_logging(topology_m4): |
| 28 |
"""Enable replication logging on all suppliers and ensure clean topology""" |
| 29 |
for server in topology_m4.ms.values(): |
| 30 |
if not server.status(): |
| 31 |
server.start() |
| 32 |
|
| 33 |
repl = ReplicationManager(DEFAULT_SUFFIX) |
| 34 |
try: |
| 35 |
for src in topology_m4.ms.values(): |
| 36 |
for dst in topology_m4.ms.values(): |
| 37 |
if src != dst: |
| 38 |
repl.ensure_agreement(src, dst) |
| 39 |
except Exception as e: |
| 40 |
log.warning(f"Could not restore full topology: {e}") |
| 41 |
pass |
| 42 |
|
| 43 |
for server in topology_m4.ms.values(): |
| 44 |
server.config.loglevel(vals=[256 + 4], service='access') |
| 45 |
server.config.loglevel(vals=[LOG_REPLICA, LOG_DEFAULT], service='error') |
| 46 |
force_log_flush(server) |
| 47 |
|
| 48 |
return topology_m4 |
| 49 |
|
| 50 |
|
| 51 |
def setup_fractional_replication(topology_m4): |
| 52 |
"""Configure fractional replication excluding telephoneNumber""" |
| 53 |
repl = ReplicationManager(DEFAULT_SUFFIX) |
| 54 |
|
| 55 |
for src in topology_m4.ms.values(): |
| 56 |
for dst in topology_m4.ms.values(): |
| 57 |
if src != dst: |
| 58 |
agmt = repl.ensure_agreement(src, dst) |
| 59 |
agmt.replace_many( |
| 60 |
('nsDS5ReplicatedAttributeListTotal', '(objectclass=*) $ EXCLUDE telephoneNumber'), |
| 61 |
('nsDS5ReplicatedAttributeList', '(objectclass=*) $ EXCLUDE telephoneNumber'), |
| 62 |
('nsds5ReplicaStripAttrs', 'modifiersname modifytimestamp'), |
| 63 |
) |
| 64 |
|
| 65 |
return repl |
| 66 |
|
| 67 |
|
| 68 |
def add_user(server, no, desc='dummy'): |
| 69 |
"""Add a test user with specific UID number""" |
| 70 |
user = UserAccounts(server, DEFAULT_SUFFIX) |
| 71 |
users = user.create_test_user(uid=no) |
| 72 |
users.add('description', [desc]) |
| 73 |
users.add('objectclass', 'userSecurityInformation') |
| 74 |
return users |
| 75 |
|
| 76 |
|
| 77 |
|
| 78 |
def count_pattern_accesslog(server, log_pattern): |
| 79 |
"""Count pattern occurrences in access log""" |
| 80 |
count = 0 |
| 81 |
server.config.set('nsslapd-accesslog-logbuffering', 'off') |
| 82 |
if server.ds_access_log.match(log_pattern): |
| 83 |
count = count + 1 |
| 84 |
return count |
| 85 |
|
| 86 |
|
| 87 |
def force_log_flush(server): |
| 88 |
"""Force server logs to flush""" |
| 89 |
server.config.set('nsslapd-accesslog-logbuffering', 'off') |
| 90 |
server.config.set('nsslapd-errorlog-logbuffering', 'off') |
| 91 |
|
| 92 |
|
| 93 |
def get_user_on_server(server, user_dn): |
| 94 |
"""Get user object on specific server, with fallback search""" |
| 95 |
try: |
| 96 |
return UserAccount(server, user_dn) |
| 97 |
except Exception: |
| 98 |
uid = user_dn.split(',')[0].split('=')[1] |
| 99 |
users = UserAccounts(server, DEFAULT_SUFFIX, rdn=None) |
| 100 |
found_users = users.filter(f'(uid={uid})') |
| 101 |
if found_users: |
| 102 |
return found_users[0] |
| 103 |
raise |
| 104 |
|
| 105 |
|
| 106 |
def test_fractional_cleanallruv_comprehensive(topology_m4, setup_replication_logging): |
| 107 |
"""Test fractional replication with cleanAllRUV operations including force and hang recovery scenarios |
| 108 |
|
| 109 |
:id: 2a68e8be-387d-4ac7-9452-1439e8483c13 |
| 110 |
:setup: Four suppliers with fractional replication excluding telephoneNumber |
| 111 |
:steps: |
| 112 |
1. Configure fractional replication excluding telephoneNumber |
| 113 |
2. Enable replication logging on all suppliers |
| 114 |
3. Create test user and verify basic replication |
| 115 |
4. Generate telephoneNumber updates (excluded attribute) to create keep alive entries |
| 116 |
5. Remove supplier3 from topology |
| 117 |
6. Execute cleanAllRUV FORCE for replica ID 3 |
| 118 |
7. Verify DEL keep alive operations are limited |
| 119 |
8. Confirm originator vs propagation behavior |
| 120 |
9. Verify replication recovery after cleanAllRUV |
| 121 |
10. Remove supplier4 from topology creating orphaned keep alive entries |
| 122 |
11. Stop supplier2 and execute non-force cleanAllRUV to trigger hang |
| 123 |
12. Verify nsds5ReplicaCleanRUV encoding on supplier1 |
| 124 |
13. Verify encoding survives supplier1 restart |
| 125 |
14. Start supplier2 and verify its encoding |
| 126 |
15. Wait for cleanAllRUV completion |
| 127 |
16. Verify final replication recovery functionality |
| 128 |
:expectedresults: |
| 129 |
1. Fractional replication configured successfully |
| 130 |
2. Logging enabled properly on all suppliers |
| 131 |
3. Basic replication works with fractional configuration |
| 132 |
4. Keep alive entries generated from excluded telephoneNumber updates |
| 133 |
5. Supplier3 removed from topology |
| 134 |
6. CleanAllRUV FORCE completes successfully |
| 135 |
7. DEL operations limited to 0 or 1 occurrences |
| 136 |
8. Supplier1 shows originator, others show propagation |
| 137 |
9. Replication continues working normally |
| 138 |
10. Supplier4 removed leaving orphaned keep alive entries |
| 139 |
11. CleanAllRUV hangs as expected without force option |
| 140 |
12. RUV encoded correctly on supplier1 |
| 141 |
13. Encoding persists across supplier1 restart |
| 142 |
14. Supplier2 shows different encoding |
| 143 |
15. CleanAllRUV completes after supplier2 restart |
| 144 |
16. Final replication recovery works correctly |
| 145 |
""" |
| 146 |
|
| 147 |
M1 = topology_m4.ms["supplier1"] |
| 148 |
M2 = topology_m4.ms["supplier2"] |
| 149 |
M3 = topology_m4.ms["supplier3"] |
| 150 |
M4 = topology_m4.ms["supplier4"] |
| 151 |
|
| 152 |
log.info('Configure fractional replication excluding telephoneNumber') |
| 153 |
repl = setup_fractional_replication(topology_m4) |
| 154 |
|
| 155 |
log.info('Create test users and verify basic replication') |
| 156 |
add_user(M1, 11, desc="add to M1") |
| 157 |
add_user(M2, 21, desc="add to M2") |
| 158 |
add_user(M3, 31, desc="add to M3") |
| 159 |
add_user(M4, 41, desc="add to M4") |
| 160 |
repl.test_replication_topology(topology_m4) |
| 161 |
|
| 162 |
log.info('Generate telephoneNumber updates (excluded attribute) to create keep alive entries') |
| 163 |
for server in topology_m4: |
| 164 |
cn = f'{USER_CN}_{11}' |
| 165 |
dn = f'uid={cn},ou=People,{DEFAULT_SUFFIX}' |
| 166 |
users = UserAccount(server, dn) |
| 167 |
for j in range(110): |
| 168 |
users.set('telephoneNumber', str(j)) |
| 169 |
|
| 170 |
repl.test_replication_topology(topology_m4) |
| 171 |
|
| 172 |
log.info('Remove supplier3 from topology') |
| 173 |
remaining_suppliers = [M1, M2, M4] |
| 174 |
repl.remove_supplier(M3, remaining_suppliers) |
| 175 |
|
| 176 |
log.info('Execute cleanAllRUV FORCE for replica ID 3') |
| 177 |
M1.tasks.cleanAllRUV(suffix=SUFFIX, replicaid='3', |
| 178 |
force=True, args={TASK_WAIT: True}) |
| 179 |
|
| 180 |
for server in remaining_suppliers: |
| 181 |
server.restart() |
| 182 |
force_log_flush(server) |
| 183 |
|
| 184 |
repl.test_replication_topology(remaining_suppliers) |
| 185 |
|
| 186 |
log.info('Verify DEL keep alive operations are limited') |
| 187 |
regex = re.compile(".*DEL dn=.cn=repl keep alive 3.*") |
| 188 |
for server in remaining_suppliers: |
| 189 |
count = count_pattern_accesslog(server, regex) |
| 190 |
log.debug(f"DEL count on {server.serverid} = {count}") |
| 191 |
assert count in (0, 1) |
| 192 |
|
| 193 |
log.info('Confirm originator vs propagation behavior') |
| 194 |
regex = re.compile(".*Original task deletes Keep alive entry .3.*") |
| 195 |
assert M1.ds_error_log.match(regex) |
| 196 |
|
| 197 |
regex = re.compile(".*Propagated task does not delete Keep alive entry .3.*") |
| 198 |
assert M2.ds_error_log.match(regex) |
| 199 |
assert M4.ds_error_log.match(regex) |
| 200 |
|
| 201 |
log.info('Verify replication recovery after cleanAllRUV') |
| 202 |
add_user(M1, 12, desc="post-cleanup M1") |
| 203 |
add_user(M2, 22, desc="post-cleanup M2") |
| 204 |
repl.test_replication_topology(remaining_suppliers) |
| 205 |
|
| 206 |
log.info('Remove supplier4 from topology creating orphaned keep alive entries') |
| 207 |
final_suppliers = [M1, M2] |
| 208 |
repl.remove_supplier(M4, final_suppliers) |
| 209 |
|
| 210 |
log.info('Stop supplier2 and execute non-force cleanAllRUV to trigger hang') |
| 211 |
M2.stop() |
| 212 |
M1.tasks.cleanAllRUV(suffix=SUFFIX, replicaid='4', force=False, args={TASK_WAIT: False}) |
| 213 |
|
| 214 |
log.info('Verify nsds5ReplicaCleanRUV encoding on supplier1') |
| 215 |
replicas = Replicas(M1) |
| 216 |
replica = replicas.list()[0] |
| 217 |
|
| 218 |
time.sleep(0.5) |
| 219 |
|
| 220 |
assert replica.present('nsds5ReplicaCleanRUV') |
| 221 |
ruv_value = replica.get_attr_val_utf8('nsds5replicacleanruv') |
| 222 |
log.info(f"M1: nsds5ReplicaCleanRUV={ruv_value}") |
| 223 |
regex_m1 = re.compile("^4:no:1:") |
| 224 |
assert regex_m1.match(ruv_value), f"M1 RUV should contain :no:1 but got: {ruv_value}" |
| 225 |
|
| 226 |
log.info('Verify encoding survives supplier1 restart') |
| 227 |
M1.restart() |
| 228 |
replicas = Replicas(M1) |
| 229 |
replica = replicas.list()[0] |
| 230 |
assert replica.present('nsds5ReplicaCleanRUV') |
| 231 |
ruv_after_restart = replica.get_attr_val_utf8('nsds5replicacleanruv') |
| 232 |
assert regex_m1.match(ruv_after_restart), f"M1 RUV changed after restart: {ruv_after_restart}" |
| 233 |
|
| 234 |
log.info('Start supplier2 and verify its encoding') |
| 235 |
M1.stop() |
| 236 |
M2.start() |
| 237 |
replicas_m2 = Replicas(M2) |
| 238 |
replica_m2 = replicas_m2.list()[0] |
| 239 |
M1.start() |
| 240 |
time.sleep(0.5) |
| 241 |
|
| 242 |
if replica_m2.present('nsds5ReplicaCleanRUV'): |
| 243 |
ruv_m2 = replica_m2.get_attr_val_utf8('nsds5replicacleanruv') |
| 244 |
log.info(f"M2: nsds5ReplicaCleanRUV={ruv_m2}") |
| 245 |
regex_m2 = re.compile("^4:no:0:") |
| 246 |
assert regex_m2.match(ruv_m2), f"M2 RUV should contain :no:0 but got: {ruv_m2}" |
| 247 |
|
| 248 |
log.info('Wait for cleanAllRUV completion') |
| 249 |
repl.test_replication_topology(final_suppliers) |
| 250 |
|
| 251 |
regex_del = re.compile(".*DEL dn=.cn=repl keep alive 4.*") |
| 252 |
for server in final_suppliers: |
| 253 |
count = count_pattern_accesslog(server, regex_del) |
| 254 |
log.info(f"DEL keep alive 4 count on {server.serverid} = {count}") |
| 255 |
assert count in (0, 1), f"Too many DEL operations on {server.serverid}: {count}" |
| 256 |
|
| 257 |
log.info('Verify final replication recovery functionality') |
| 258 |
user_13 = add_user(M1, 13, desc="final-cleanup M1") |
| 259 |
user_23 = add_user(M2, 23, desc="final-cleanup M2") |
| 260 |
|
| 261 |
repl.test_replication_topology(final_suppliers) |
| 262 |
|
| 263 |
for server in final_suppliers: |
| 264 |
user_13_obj = UserAccount(server, user_13.dn) |
| 265 |
assert user_13_obj.exists(), f"User test_user_13 NOT found on {server.serverid}" |
| 266 |
|
| 267 |
user_23_obj = UserAccount(server, user_23.dn) |
| 268 |
assert user_23_obj.exists(), f"User test_user_23 NOT found on {server.serverid}" |
| 269 |
|
| 270 |
|
| 271 |
if __name__ == '__main__': |
| 272 |
CURRENT_FILE = os.path.realpath(__file__) |
| 273 |
pytest.main(["-s", CURRENT_FILE]) |