Debian Codegraph @

dirsrvtests/tests/suites/replication/cleanallruv_fractional_test.py

raw
1 # --- BEGIN COPYRIGHT BLOCK ---
2 # Copyright (C) 2025 Red Hat, Inc.
3 # All rights reserved.
4 #
5 # License: GPL (version 3 or any later version).
6 # See LICENSE for details.
7 # --- END COPYRIGHT BLOCK ---
8 #
9 import logging
10 import pytest
11 import os
12 import re
13 import time
14 from lib389._constants import DEFAULT_SUFFIX, SUFFIX, LOG_REPLICA, LOG_DEFAULT
15 from test389.topologies import topology_m4
16 from lib389.replica import Replicas, ReplicationManager
17 from lib389.idm.user import UserAccounts, UserAccount
18 from lib389.properties import TASK_WAIT
19
20 pytestmark = pytest.mark.tier2
21
22 USER_CN = "test_user"
23 log = logging.getLogger(__name__)
24
25
26 @pytest.fixture(scope="function")
27 def setup_replication_logging(topology_m4):
28 """Enable replication logging on all suppliers and ensure clean topology"""
29 for server in topology_m4.ms.values():
30 if not server.status():
31 server.start()
32
33 repl = ReplicationManager(DEFAULT_SUFFIX)
34 try:
35 for src in topology_m4.ms.values():
36 for dst in topology_m4.ms.values():
37 if src != dst:
38 repl.ensure_agreement(src, dst)
39 except Exception as e:
40 log.warning(f"Could not restore full topology: {e}")
41 pass
42
43 for server in topology_m4.ms.values():
44 server.config.loglevel(vals=[256 + 4], service='access')
45 server.config.loglevel(vals=[LOG_REPLICA, LOG_DEFAULT], service='error')
46 force_log_flush(server)
47
48 return topology_m4
49
50
51 def setup_fractional_replication(topology_m4):
52 """Configure fractional replication excluding telephoneNumber"""
53 repl = ReplicationManager(DEFAULT_SUFFIX)
54
55 for src in topology_m4.ms.values():
56 for dst in topology_m4.ms.values():
57 if src != dst:
58 agmt = repl.ensure_agreement(src, dst)
59 agmt.replace_many(
60 ('nsDS5ReplicatedAttributeListTotal', '(objectclass=*) $ EXCLUDE telephoneNumber'),
61 ('nsDS5ReplicatedAttributeList', '(objectclass=*) $ EXCLUDE telephoneNumber'),
62 ('nsds5ReplicaStripAttrs', 'modifiersname modifytimestamp'),
63 )
64
65 return repl
66
67
68 def add_user(server, no, desc='dummy'):
69 """Add a test user with specific UID number"""
70 user = UserAccounts(server, DEFAULT_SUFFIX)
71 users = user.create_test_user(uid=no)
72 users.add('description', [desc])
73 users.add('objectclass', 'userSecurityInformation')
74 return users
75
76
77
78 def count_pattern_accesslog(server, log_pattern):
79 """Count pattern occurrences in access log"""
80 count = 0
81 server.config.set('nsslapd-accesslog-logbuffering', 'off')
82 if server.ds_access_log.match(log_pattern):
83 count = count + 1
84 return count
85
86
87 def force_log_flush(server):
88 """Force server logs to flush"""
89 server.config.set('nsslapd-accesslog-logbuffering', 'off')
90 server.config.set('nsslapd-errorlog-logbuffering', 'off')
91
92
93 def get_user_on_server(server, user_dn):
94 """Get user object on specific server, with fallback search"""
95 try:
96 return UserAccount(server, user_dn)
97 except Exception:
98 uid = user_dn.split(',')[0].split('=')[1]
99 users = UserAccounts(server, DEFAULT_SUFFIX, rdn=None)
100 found_users = users.filter(f'(uid={uid})')
101 if found_users:
102 return found_users[0]
103 raise
104
105
106 def test_fractional_cleanallruv_comprehensive(topology_m4, setup_replication_logging):
107 """Test fractional replication with cleanAllRUV operations including force and hang recovery scenarios
108
109 :id: 2a68e8be-387d-4ac7-9452-1439e8483c13
110 :setup: Four suppliers with fractional replication excluding telephoneNumber
111 :steps:
112 1. Configure fractional replication excluding telephoneNumber
113 2. Enable replication logging on all suppliers
114 3. Create test user and verify basic replication
115 4. Generate telephoneNumber updates (excluded attribute) to create keep alive entries
116 5. Remove supplier3 from topology
117 6. Execute cleanAllRUV FORCE for replica ID 3
118 7. Verify DEL keep alive operations are limited
119 8. Confirm originator vs propagation behavior
120 9. Verify replication recovery after cleanAllRUV
121 10. Remove supplier4 from topology creating orphaned keep alive entries
122 11. Stop supplier2 and execute non-force cleanAllRUV to trigger hang
123 12. Verify nsds5ReplicaCleanRUV encoding on supplier1
124 13. Verify encoding survives supplier1 restart
125 14. Start supplier2 and verify its encoding
126 15. Wait for cleanAllRUV completion
127 16. Verify final replication recovery functionality
128 :expectedresults:
129 1. Fractional replication configured successfully
130 2. Logging enabled properly on all suppliers
131 3. Basic replication works with fractional configuration
132 4. Keep alive entries generated from excluded telephoneNumber updates
133 5. Supplier3 removed from topology
134 6. CleanAllRUV FORCE completes successfully
135 7. DEL operations limited to 0 or 1 occurrences
136 8. Supplier1 shows originator, others show propagation
137 9. Replication continues working normally
138 10. Supplier4 removed leaving orphaned keep alive entries
139 11. CleanAllRUV hangs as expected without force option
140 12. RUV encoded correctly on supplier1
141 13. Encoding persists across supplier1 restart
142 14. Supplier2 shows different encoding
143 15. CleanAllRUV completes after supplier2 restart
144 16. Final replication recovery works correctly
145 """
146
147 M1 = topology_m4.ms["supplier1"]
148 M2 = topology_m4.ms["supplier2"]
149 M3 = topology_m4.ms["supplier3"]
150 M4 = topology_m4.ms["supplier4"]
151
152 log.info('Configure fractional replication excluding telephoneNumber')
153 repl = setup_fractional_replication(topology_m4)
154
155 log.info('Create test users and verify basic replication')
156 add_user(M1, 11, desc="add to M1")
157 add_user(M2, 21, desc="add to M2")
158 add_user(M3, 31, desc="add to M3")
159 add_user(M4, 41, desc="add to M4")
160 repl.test_replication_topology(topology_m4)
161
162 log.info('Generate telephoneNumber updates (excluded attribute) to create keep alive entries')
163 for server in topology_m4:
164 cn = f'{USER_CN}_{11}'
165 dn = f'uid={cn},ou=People,{DEFAULT_SUFFIX}'
166 users = UserAccount(server, dn)
167 for j in range(110):
168 users.set('telephoneNumber', str(j))
169
170 repl.test_replication_topology(topology_m4)
171
172 log.info('Remove supplier3 from topology')
173 remaining_suppliers = [M1, M2, M4]
174 repl.remove_supplier(M3, remaining_suppliers)
175
176 log.info('Execute cleanAllRUV FORCE for replica ID 3')
177 M1.tasks.cleanAllRUV(suffix=SUFFIX, replicaid='3',
178 force=True, args={TASK_WAIT: True})
179
180 for server in remaining_suppliers:
181 server.restart()
182 force_log_flush(server)
183
184 repl.test_replication_topology(remaining_suppliers)
185
186 log.info('Verify DEL keep alive operations are limited')
187 regex = re.compile(".*DEL dn=.cn=repl keep alive 3.*")
188 for server in remaining_suppliers:
189 count = count_pattern_accesslog(server, regex)
190 log.debug(f"DEL count on {server.serverid} = {count}")
191 assert count in (0, 1)
192
193 log.info('Confirm originator vs propagation behavior')
194 regex = re.compile(".*Original task deletes Keep alive entry .3.*")
195 assert M1.ds_error_log.match(regex)
196
197 regex = re.compile(".*Propagated task does not delete Keep alive entry .3.*")
198 assert M2.ds_error_log.match(regex)
199 assert M4.ds_error_log.match(regex)
200
201 log.info('Verify replication recovery after cleanAllRUV')
202 add_user(M1, 12, desc="post-cleanup M1")
203 add_user(M2, 22, desc="post-cleanup M2")
204 repl.test_replication_topology(remaining_suppliers)
205
206 log.info('Remove supplier4 from topology creating orphaned keep alive entries')
207 final_suppliers = [M1, M2]
208 repl.remove_supplier(M4, final_suppliers)
209
210 log.info('Stop supplier2 and execute non-force cleanAllRUV to trigger hang')
211 M2.stop()
212 M1.tasks.cleanAllRUV(suffix=SUFFIX, replicaid='4', force=False, args={TASK_WAIT: False})
213
214 log.info('Verify nsds5ReplicaCleanRUV encoding on supplier1')
215 replicas = Replicas(M1)
216 replica = replicas.list()[0]
217
218 time.sleep(0.5)
219
220 assert replica.present('nsds5ReplicaCleanRUV')
221 ruv_value = replica.get_attr_val_utf8('nsds5replicacleanruv')
222 log.info(f"M1: nsds5ReplicaCleanRUV={ruv_value}")
223 regex_m1 = re.compile("^4:no:1:")
224 assert regex_m1.match(ruv_value), f"M1 RUV should contain :no:1 but got: {ruv_value}"
225
226 log.info('Verify encoding survives supplier1 restart')
227 M1.restart()
228 replicas = Replicas(M1)
229 replica = replicas.list()[0]
230 assert replica.present('nsds5ReplicaCleanRUV')
231 ruv_after_restart = replica.get_attr_val_utf8('nsds5replicacleanruv')
232 assert regex_m1.match(ruv_after_restart), f"M1 RUV changed after restart: {ruv_after_restart}"
233
234 log.info('Start supplier2 and verify its encoding')
235 M1.stop()
236 M2.start()
237 replicas_m2 = Replicas(M2)
238 replica_m2 = replicas_m2.list()[0]
239 M1.start()
240 time.sleep(0.5)
241
242 if replica_m2.present('nsds5ReplicaCleanRUV'):
243 ruv_m2 = replica_m2.get_attr_val_utf8('nsds5replicacleanruv')
244 log.info(f"M2: nsds5ReplicaCleanRUV={ruv_m2}")
245 regex_m2 = re.compile("^4:no:0:")
246 assert regex_m2.match(ruv_m2), f"M2 RUV should contain :no:0 but got: {ruv_m2}"
247
248 log.info('Wait for cleanAllRUV completion')
249 repl.test_replication_topology(final_suppliers)
250
251 regex_del = re.compile(".*DEL dn=.cn=repl keep alive 4.*")
252 for server in final_suppliers:
253 count = count_pattern_accesslog(server, regex_del)
254 log.info(f"DEL keep alive 4 count on {server.serverid} = {count}")
255 assert count in (0, 1), f"Too many DEL operations on {server.serverid}: {count}"
256
257 log.info('Verify final replication recovery functionality')
258 user_13 = add_user(M1, 13, desc="final-cleanup M1")
259 user_23 = add_user(M2, 23, desc="final-cleanup M2")
260
261 repl.test_replication_topology(final_suppliers)
262
263 for server in final_suppliers:
264 user_13_obj = UserAccount(server, user_13.dn)
265 assert user_13_obj.exists(), f"User test_user_13 NOT found on {server.serverid}"
266
267 user_23_obj = UserAccount(server, user_23.dn)
268 assert user_23_obj.exists(), f"User test_user_23 NOT found on {server.serverid}"
269
270
271 if __name__ == '__main__':
272 CURRENT_FILE = os.path.realpath(__file__)
273 pytest.main(["-s", CURRENT_FILE])