Debian Codegraph @

dirsrvtests/tests/suites/replication/changelog_trimming_test.py

raw
1 # --- BEGIN COPYRIGHT BLOCK ---
2 # Copyright (C) 2022 Red Hat, Inc.
3 # All rights reserved.
4 #
5 # License: GPL (version 3 or any later version).
6 # See LICENSE for details.
7 # --- END COPYRIGHT BLOCK ---
8 #
9 import logging
10 import pytest
11 import os
12 import ldap
13 import time
14 from lib389._constants import *
15 from lib389.properties import *
16 from test389.topologies import topology_m1 as topo
17 from test389.topologies import topology_m1c1
18 from lib389.tasks import *
19 from lib389.replica import Changelog5
20 from lib389.idm.domain import Domain
21 from lib389.idm.user import UserAccounts
22 from lib389.utils import ensure_bytes, ds_supports_new_changelog
23 from lib389.replica import ReplicationManager
24 from lib389._constants import DN_LDBM
25
26 pytestmark = pytest.mark.tier1
27
28 DEBUGGING = os.getenv("DEBUGGING", default=False)
29 if DEBUGGING:
30 logging.getLogger(__name__).setLevel(logging.DEBUG)
31 else:
32 logging.getLogger(__name__).setLevel(logging.INFO)
33 log = logging.getLogger(__name__)
34
35 CHANGELOG = 'cn=changelog,{}'.format(DN_USERROOT_LDBM)
36 MAXAGE = 'nsslapd-changelogmaxage'
37 MAXENTRIES = 'nsslapd-changelogmaxentries'
38 TRIMINTERVAL = 'nsslapd-changelogtrim-interval'
39
40 def do_mods(supplier, num):
41 """Perform a num of mods on the default suffix
42 """
43 domain = Domain(supplier, DEFAULT_SUFFIX)
44 for i in range(num):
45 domain.replace('description', 'change %s' % i)
46
47 def set_value(supplier, attr, val):
48 """
49 Helper function to add/replace attr: val and check the added value
50 """
51 try:
52 supplier.modify_s(CHANGELOG, [(ldap.MOD_REPLACE, attr, ensure_bytes(val))])
53 except ldap.LDAPError as e:
54 log.error('Failed to add ' + attr + ': ' + val + ' to ' + plugin + ': error {}'.format(get_ldap_error_msg(e,'desc')))
55 assert False
56
57 @pytest.fixture(scope="module")
58 def setup_max_entries(topo, request):
59 """Configure logging and changelog max entries
60 """
61 supplier = topo.ms["supplier1"]
62
63 supplier.config.loglevel((ErrorLog.REPLICA,), 'error')
64
65 if ds_supports_new_changelog():
66 set_value(supplier, MAXENTRIES, '2')
67 set_value(supplier, TRIMINTERVAL, '300')
68 else:
69 cl = Changelog5(supplier)
70 cl.set_trim_interval('300')
71
72 @pytest.fixture(scope="module")
73 def setup_max_age(topo, request):
74 """Configure logging and changelog max age
75 """
76 supplier = topo.ms["supplier1"]
77 supplier.config.loglevel((ErrorLog.REPLICA,), 'error')
78
79 if ds_supports_new_changelog():
80 set_value(supplier, MAXAGE, '5s')
81 set_value(supplier, TRIMINTERVAL, '300')
82 else:
83 cl = Changelog5(supplier)
84 cl.set_max_age('5s')
85 cl.set_trim_interval('300')
86
87 def test_max_age(topo, setup_max_age):
88 """Test changing the trimming interval works with max age
89
90 :id: b5de04a5-4d92-49ea-a725-1d278a1c647c
91 :setup: single supplier
92 :steps:
93 1. Perform modification to populate changelog
94 2. Adjust the changelog trimming interval
95 3. Check is trimming occurrs within the new interval
96
97 :expectedresults:
98 1. Modifications are successful
99 2. The changelog trimming interval is correctly lowered
100 3. Trimming occurs
101
102 """
103 log.info("Testing changelog trimming interval with max age...")
104
105 supplier = topo.ms["supplier1"]
106 if not ds_supports_new_changelog():
107 cl = Changelog5(supplier)
108
109 # Do mods to build if cl entries
110 do_mods(supplier, 10)
111
112 time.sleep(1) # Trimming should not have occurred
113 if supplier.searchErrorsLog("trimmed") is True:
114 log.fatal('Trimming event unexpectedly occurred')
115 assert False
116
117 if ds_supports_new_changelog():
118 set_value(supplier, TRIMINTERVAL, '5')
119 else:
120 cl.set_trim_interval('5')
121
122 time.sleep(3) # Trimming should not have occurred
123 if supplier.searchErrorsLog("trimmed") is True:
124 log.fatal('Trimming event unexpectedly occurred')
125 assert False
126
127 time.sleep(3) # Trimming should have occurred
128 if supplier.searchErrorsLog("trimmed") is False:
129 log.fatal('Trimming event did not occur')
130 assert False
131
132
133 def test_max_entries(topo, setup_max_entries):
134 """Test changing the trimming interval works with max entries
135
136 :id: b5de04a5-4d92-49ea-a725-1d278a1c647d
137 :setup: single supplier
138 :steps:
139 1. Perform modification to populate changelog
140 2. Adjust the changelog trimming interval
141 3. Check is trimming occurrs within the new interval
142
143 :expectedresults:
144 1. Modifications are successful
145 2. The changelog trimming interval is correctly lowered
146 3. Trimming occurs
147
148 """
149
150 log.info("Testing changelog triming interval with max entries...")
151 supplier = topo.ms["supplier1"]
152 if not ds_supports_new_changelog():
153 cl = Changelog5(supplier)
154
155 # reset errors log
156 supplier.deleteErrorLogs()
157
158 # Do mods to build if cl entries
159 do_mods(supplier, 10)
160
161 time.sleep(1) # Trimming should have occurred
162 if supplier.searchErrorsLog("trimmed") is True:
163 log.fatal('Trimming event unexpectedly occurred')
164 assert False
165
166 if ds_supports_new_changelog():
167 set_value(supplier, TRIMINTERVAL, '5')
168 else:
169 cl.set_trim_interval('5')
170
171 time.sleep(6) # Trimming should have occurred
172 if supplier.searchErrorsLog("trimmed") is False:
173 log.fatal('Trimming event did not occur')
174 assert False
175
176 def test_cl_trim_ignore_empty_ruv_element(topology_m1c1):
177 """Test trimming is not done on RID not yet replicated
178
179 :id: b4b42d8d-dfd1-482e-aca4-8f484761c541
180 :setup: single supplier and single consumer
181 :steps:
182 1. Initialize S1 with new dataset (S1 and C1 are out of sync)
183 2. Export LDIF the replicated dataset from S1. This LDIF does contain an empty RUV
184 3. Do a dummy update on S1 (still not replicated to C1)
185 4. Stop S1 so that it will not try to update C1
186 5. Reinit C1 from LDIF from step 2
187 6. Switch to readonly mode the default backend so that it will NOT apply any change (direct or replicated)
188 7. Start S1, as C1 is in readlonly
189 8. Adjust trim interval/maxage on S1
190 9. Wait for maxage+5s for the trimming to occur
191 10. Switch to not readlonly the default backend so that it will apply updates sent by S1
192 11. Check that C1 catch up and a new update is also replicated
193 12. Check S1 had not trim the starting point CSN as we can not find the message "Can't locate CSN"
194
195 :expectedresults:
196 1. successfull export/import on S1
197 2. successfull export on S1
198 3. successfull update
199 4. success
200 5. successfull init of C1
201 6. successfull enable readonly
202 7. success
203 8. successfull setting for trimming
204 9. success
205 10. successfull disable readonly
206 11. success
207 12. message not found
208
209 """
210 log.info("Test trimming is not done on RID not yet replicated...")
211
212 s1 = topology_m1c1.ms['supplier1']
213 s1_tasks = Tasks(s1)
214 s1_tasks.log = log
215 c1 = topology_m1c1.cs['consumer1']
216 c1_tasks = Tasks(c1)
217 c1_tasks.log = log
218
219 # Step 1 - reinit (export/import) S1 with an empty RUV
220 ldif_file = "{}/export.ldif".format(s1.get_ldif_dir())
221 args = {EXPORT_REPL_INFO: False, # to get a ldif without RUV
222 TASK_WAIT: True}
223 s1_tasks.exportLDIF(DEFAULT_SUFFIX, None, ldif_file, args)
224 args = {TASK_WAIT: True}
225 s1_tasks.importLDIF(DEFAULT_SUFFIX, None, ldif_file, args)
226
227 # Step 2 - At this point S1 and C1 are out of sync as we reinited S1
228 # with new dataset
229 # Export the DB (with empty RUV) to be used to reinit C1
230 ldif_file = "{}/export_empty_ruv.ldif".format(s1.get_ldif_dir())
231 args = {EXPORT_REPL_INFO: True,
232 TASK_WAIT: True}
233 s1_tasks.exportLDIF(DEFAULT_SUFFIX, None, ldif_file, args)
234
235 # - Step 3 - Dummy update on s1 to kick the RUV
236 test_users_s1 = UserAccounts(s1, DEFAULT_SUFFIX, rdn=None)
237 user_1 = test_users_s1.create_test_user(uid=1000)
238
239 # - Step 4 - Stop supplier1 to prevent it to update C1 once this one
240 s1.stop()
241
242 # - Step 5 - reinit C1 with empty RUV ldif file
243 args = {TASK_WAIT: True}
244 c1_tasks.importLDIF(DEFAULT_SUFFIX, None, ldif_file, args)
245
246 # - Step 6 - Put default backend of C1 into read only mode
247 c1.modify_s('cn=%s,%s' % (DEFAULT_BENAME, DN_LDBM), [(ldap.MOD_REPLACE, 'nsslapd-readonly', b'on')])
248
249 # - Step 7 - Now it is safe to start S1, no update will apply on C1
250 # and C1 RUV will stay empty
251 s1.start()
252
253 # - Step 8 - set Triming interval to 2sec to have frequent trimming
254 # and max-age to 8sec to not wait too long
255 if ds_supports_new_changelog():
256 set_value(s1, MAXAGE, '8s')
257 set_value(s1, TRIMINTERVAL, '2')
258 else:
259 cl = Changelog5(s1)
260 cl.set_max_age('8s')
261 cl.set_trim_interval('2')
262
263 # - Step 9 - Wait for the trimming to occur
264 time.sleep(30)
265
266 # - Step 10 - disable read-only mode on the default backend of C1
267 c1.modify_s('cn=%s,%s' % (DEFAULT_BENAME, DN_LDBM), [(ldap.MOD_REPLACE, 'nsslapd-readonly', b'off')])
268
269 # - Step 11 - Check that M1 and M2 are in sync
270 repl = ReplicationManager(DEFAULT_SUFFIX)
271 repl.wait_for_replication(s1, c1, timeout=20)
272
273 # Recheck after another update
274 user_2 = test_users_s1.create_test_user(uid=2000)
275 repl.wait_for_replication(s1, c1, timeout=20)
276
277 # - Step 12 - check in infamous message when a CSN
278 # is missing from the changelog
279 assert not s1.ds_error_log.match('.*Can.t locate CSN}.*')
280
281 if __name__ == '__main__':
282 # Run isolated
283 # -s for DEBUG mode
284 CURRENT_FILE = os.path.realpath(__file__)
285 pytest.main("-s %s" % CURRENT_FILE)