Debian Codegraph @

dirsrvtests/tests/suites/clu/dsconf_tasks_test.py

raw
1 # --- BEGIN COPYRIGHT BLOCK ---
2 # Copyright (C) 2023 Red Hat, Inc.
3 # All rights reserved.
4 #
5 # License: GPL (version 3 or any later version).
6 # See LICENSE for details.
7 # --- END COPYRIGHT BLOCK ---
8 #
9 import logging
10 import os
11 import subprocess
12 import uuid
13
14 import pytest
15
16 from lib389._constants import DEFAULT_SUFFIX, DN_DM
17 # from test389.topologies import topology_m1 as topo
18 from test389.topologies import topology_st as topo
19 from lib389.tasks import (ImportTask, ExportTask, BackupTask, RestoreTask, AutomemberRebuildMembershipTask,
20 AutomemberAbortRebuildTask, MemberUidFixupTask, MemberOfFixupTask, USNTombstoneCleanupTask,
21 DBCompactTask, EntryUUIDFixupTask, SchemaReloadTask, SyntaxValidateTask,
22 FixupLinkedAttributesTask)
23 from lib389.plugins import USNPlugin, POSIXWinsyncPlugin, LinkedAttributesPlugin, AutoMembershipPlugin, MemberOfPlugin
24 from lib389.dbgen import dbgen_users
25 from lib389.idm.user import UserAccount
26 from lib389.idm.group import Groups
27 from lib389.idm.posixgroup import PosixGroups
28
29 log = logging.getLogger(__name__)
30
31 DSCONF = '/usr/sbin/dsconf'
32 # Enough entries for import/export/reindex to emit multiple nsTaskLog updates when --watch is used.
33 WATCH_TEST_NUM_USERS = 20000
34
35
36 def _dsconf_base_cmd(inst):
37 return [DSCONF, inst.serverid, '-D', DN_DM, '-w', 'password']
38
39
40 def _run_dsconf(inst, argv, timeout=7200):
41 """Run dsconf; return (returncode, combined stdout+stderr)."""
42 cmd = _dsconf_base_cmd(inst) + argv
43 proc = subprocess.run(
44 cmd,
45 capture_output=True,
46 text=True,
47 timeout=timeout,
48 )
49 out = (proc.stdout or '') + (proc.stderr or '')
50 return proc.returncode, out
51
52
53 def _non_empty_lines(text):
54 return [ln for ln in text.splitlines() if ln.strip()]
55
56
57 def test_dsconf_task_watch_output(topo):
58 """dsconf backend / backup tasks: ``--watch`` yields more output than default
59
60 Verifies that ``task.watch()``-driven progress (incremental nsTaskLog lines) appears when
61 ``-w`` / ``--watch`` is passed, and that the non-watch paths stay comparatively minimal.
62
63 Reindex is compared with ``--wait`` alone versus ``--wait --watch``; without ``--wait``,
64 ``backend index reindex`` returns immediately and does not wait for the task.
65
66 Backup ``create`` / ``restore`` use ``--timeout 0`` so the default 120s task limit does not
67 fire on a large database.
68
69 :id: 7c4e8a91-2b0d-4f6e-9c1d-8a3e5f7b0d2c
70 :setup: Standalone Instance
71 :steps:
72 1. Online-import a large LDIF without ``--watch``; capture output line count
73 2. Re-import the same LDIF with ``--watch``; capture output — expect more lines
74 3. Export without ``--watch``; then export with ``--watch`` — expect more lines on watch
75 4. ``backend index reindex`` with ``--wait`` only vs ``--wait --watch`` — expect more lines with watch
76 5. ``backup create`` to a dedicated directory without ``--watch``, then to another with ``--watch``
77 6. ``backup restore`` from each archive without / with ``--watch``
78 :expectedresults:
79 1. Import succeeds; non-watch output has few lines
80 2. Import with watch succeeds; strictly more non-empty log lines than without watch
81 3. Both exports succeed; watch run is more verbose
82 4. Both reindex runs succeed; watch run is more verbose
83 5. Both backup creates succeed; watch run is more verbose
84 6. Both restores succeed; watch run is more verbose
85 """
86 inst = topo.standalone
87 be = 'userRoot'
88 tag = uuid.uuid4().hex[:12]
89 import_ldif = os.path.join(inst.ldifdir, 'dsconf_watch_test_import.ldif')
90 export_ldif_a = os.path.join(inst.ldifdir, 'dsconf_watch_test_export_a.ldif')
91 export_ldif_b = os.path.join(inst.ldifdir, 'dsconf_watch_test_export_b.ldif')
92
93 dbgen_users(
94 inst,
95 WATCH_TEST_NUM_USERS,
96 import_ldif,
97 DEFAULT_SUFFIX,
98 parent='ou=people,' + DEFAULT_SUFFIX,
99 generic=True,
100 )
101
102 # import
103 rc, out_imp_nowatch = _run_dsconf(
104 inst,
105 ['backend', 'import', be, import_ldif],
106 )
107 assert rc == 0, out_imp_nowatch
108 lines_imp_nowatch = len(_non_empty_lines(out_imp_nowatch))
109 assert "Processing file" not in out_imp_nowatch, "'Processing file' should not be in the output"
110
111 rc, out_imp_watch = _run_dsconf(
112 inst,
113 ['backend', 'import', be, import_ldif, '--watch'],
114 )
115 assert rc == 0, out_imp_watch
116 lines_imp_watch = len(_non_empty_lines(out_imp_watch))
117 assert lines_imp_watch > lines_imp_nowatch, (
118 f'import --watch should log more lines than without (got {lines_imp_watch} vs {lines_imp_nowatch})'
119 )
120 assert "Processing file" in out_imp_watch, "'Processing file' should be in the output"
121
122 # export
123 rc, out_exp_nowatch = _run_dsconf(
124 inst,
125 ['backend', 'export', be, '-l', export_ldif_a],
126 )
127 assert rc == 0, out_exp_nowatch
128 lines_exp_nowatch = len(_non_empty_lines(out_exp_nowatch))
129 assert "userRoot: Processed" not in out_exp_nowatch, "'userRoot: Processed' should not be in the output"
130
131 rc, out_exp_watch = _run_dsconf(
132 inst,
133 ['backend', 'export', be, '-l', export_ldif_b, '--watch'],
134 )
135 assert rc == 0, out_exp_watch
136 lines_exp_watch = len(_non_empty_lines(out_exp_watch))
137 assert lines_exp_watch > lines_exp_nowatch, (
138 f'export --watch should log more lines than without (got {lines_exp_watch} vs {lines_exp_nowatch})'
139 )
140 assert "userRoot: Processed" in out_exp_watch, "'userRoot: Processed' should be in the output"
141
142 # reindex
143 rc, out_idx_nowatch = _run_dsconf(
144 inst,
145 ['backend', 'index', 'reindex', be, '--wait'],
146 )
147 assert rc == 0, out_idx_nowatch
148 lines_idx_nowatch = len(_non_empty_lines(out_idx_nowatch))
149 assert "Indexing attribute:" not in out_idx_nowatch, "'Indexing attribute:' should not be in the output"
150
151 rc, out_idx_watch = _run_dsconf(
152 inst,
153 ['backend', 'index', 'reindex', be, '--wait', '--watch'],
154 )
155 assert rc == 0, out_idx_watch
156 lines_idx_watch = len(_non_empty_lines(out_idx_watch))
157 assert lines_idx_watch > lines_idx_nowatch, (
158 f'reindex --watch should log more lines than --wait alone (got {lines_idx_watch} vs {lines_idx_nowatch})'
159 )
160 assert "Indexing attribute:" in out_idx_watch, "'Indexing attribute:' should be in the output"
161
162 # backup create
163 bak_create_a = os.path.join(inst.ds_paths.backup_dir, f'dsconf_watch_backup_create_a_{tag}')
164 bak_create_b = os.path.join(inst.ds_paths.backup_dir, f'dsconf_watch_backup_create_b_{tag}')
165
166 rc, out_bak_c_nowatch = _run_dsconf(
167 inst,
168 ['backup', 'create', bak_create_a],
169 )
170 assert rc == 0, out_bak_c_nowatch
171 lines_bak_c_nowatch = len(_non_empty_lines(out_bak_c_nowatch))
172 assert "Creating backup" not in out_bak_c_nowatch, "'Creating backup' should not be in the output"
173
174 rc, out_bak_c_watch = _run_dsconf(
175 inst,
176 ['backup', 'create', bak_create_b, '--watch'],
177 )
178 assert rc == 0, out_bak_c_watch
179 lines_bak_c_watch = len(_non_empty_lines(out_bak_c_watch))
180 assert lines_bak_c_watch > lines_bak_c_nowatch, (
181 f'backup create --watch should log more lines (got {lines_bak_c_watch} vs {lines_bak_c_nowatch})'
182 )
183 assert "Creating backup" in out_bak_c_watch, "'Creating backup' should be in the output"
184
185 # backup restore
186 rc, out_bak_r_nowatch = _run_dsconf(
187 inst,
188 ['backup', 'restore', bak_create_a, '--timeout', '0'],
189 )
190 assert rc == 0, out_bak_r_nowatch
191 lines_bak_r_nowatch = len(_non_empty_lines(out_bak_r_nowatch))
192 assert "Restoring backup" not in out_bak_r_nowatch, "'Restoring backup' should not be in the output"
193
194 rc, out_bak_r_watch = _run_dsconf(
195 inst,
196 ['backup', 'restore', bak_create_b, '--timeout', '0', '--watch'],
197 )
198 assert rc == 0, out_bak_r_watch
199 lines_bak_r_watch = len(_non_empty_lines(out_bak_r_watch))
200 assert lines_bak_r_watch > lines_bak_r_nowatch, (
201 f'backup restore --watch should log more lines (got {lines_bak_r_watch} vs {lines_bak_r_nowatch})'
202 )
203 assert "Restoring backup" in out_bak_r_watch, "'Restoring backup' should be in the output"
204
205
206 def test_task_timeout(topo):
207 """All thath te timeoutsetting works for all "tasks"
208
209 :id: 6a6f5176-76bf-424d-bc10-d33bdfa529eb
210 :setup: Standalone Instance
211 :steps:
212 1. Test timeout for import task
213 2. Test timeout for export task
214 3. Test timeout for schema validate task
215 4. Test timeout for schema reload task
216 5. Test timeout for automember rebuild
217 6. Test timeout for automember abort
218 7. Test timeout for usn cleanup task
219 8. Test timeout for posix group fixup task
220 9. Test timeout for member UID fixup task
221 10. Test timeout for memberof fixup task
222 11. Test timeout for entryuuid fixup task
223 12. Test timeout for linked attrs fixup task
224 13. test timeout for db compact task
225 :expectedresults:
226 1. Task timed out
227 2. Task timed out
228 3. Task timed out
229 4. Task timed out
230 5. Task timed out
231 6. Task timed out
232 7. Task timed out
233 8. Task timed out
234 9. Task timed out
235 10. Task timed out
236 11. Task timed out
237 12. Task timed out
238 13. Task timed out
239 """
240
241 #inst = topo.ms['supplier1'] --> this leads to a deadlock when testing MemberOfFixupTask
242 inst = topo.standalone
243
244 # Enable plugins
245 plugins = [USNPlugin, POSIXWinsyncPlugin, LinkedAttributesPlugin, AutoMembershipPlugin, MemberOfPlugin]
246 for plugin in plugins:
247 plugin(inst).enable()
248 inst.restart()
249
250 # Test timeout for import task, first create LDIF
251 import_ldif = inst.ldifdir + '/import_task_timeout.ldif'
252 dbgen_users(inst, 100000, import_ldif, DEFAULT_SUFFIX, parent="ou=people," + DEFAULT_SUFFIX, generic=True)
253
254 task = ImportTask(inst)
255 task.import_suffix_from_ldif(ldiffile=import_ldif, suffix=DEFAULT_SUFFIX)
256 task.wait(timeout=.5, sleep_interval=.5)
257 assert task.get_exit_code() is None
258 task.wait(timeout=0)
259
260 # Test timeout for export task
261 export_ldif = inst.ldifdir + '/export_task_timeout.ldif'
262 task = ExportTask(inst)
263 task.export_suffix_to_ldif(export_ldif, DEFAULT_SUFFIX)
264 task.wait(timeout=.5, sleep_interval=.5)
265 if task.get_exit_code() is None:
266 task.wait(timeout=0)
267 else:
268 log.info('Export task completed before timeout - skipping timeout check')
269 assert task.get_exit_code() == 0
270
271 # Test timeout for schema validate task
272 task = SyntaxValidateTask(inst).create(properties={
273 'basedn': DEFAULT_SUFFIX,
274 'filter': "objectClass=*"
275 })
276 task.wait(timeout=.5, sleep_interval=.5)
277 assert task.get_exit_code() is None
278 task.wait(timeout=0)
279
280 # Test timeout for schema reload task (runs too fast)
281 """
282 task = SchemaReloadTask(inst).create(properties={
283 'schemadir': inst.schemadir,
284 })
285 task.wait(timeout=.5, sleep_interval=.5)
286 assert task.get_exit_code() is None
287 task.wait(timeout=0)
288 """
289
290 # Test timeout for automember rebuild
291 task = AutomemberRebuildMembershipTask(inst).create(properties={
292 'basedn': DEFAULT_SUFFIX,
293 'filter': "objectClass=*"
294 })
295 task.wait(timeout=.5, sleep_interval=.5)
296 assert task.get_exit_code() is None
297 task.wait(timeout=0)
298
299 # Test timeout for automember abort (runs too fast)
300 """
301 AutomemberRebuildMembershipTask(inst).create(properties={
302 'basedn': DEFAULT_SUFFIX,
303 'filter': "objectClass=*"
304 })
305 task = AutomemberAbortRebuildTask(inst).create()
306 task.wait(timeout=.5, sleep_interval=.5)
307 assert task.get_exit_code() is None
308 task.wait(timeout=0)
309 """
310
311 # Test timeout for usn cleanup task, first delete a bunch of users
312 for idx in range(1, 1001):
313 entry_idx = str(idx).zfill(6)
314 dn = f"uid=user{entry_idx},ou=people,{DEFAULT_SUFFIX}"
315 UserAccount(inst, dn=dn).delete()
316 task = USNTombstoneCleanupTask(inst).create(properties={
317 'suffix': DEFAULT_SUFFIX,
318 })
319 task.wait(timeout=.5, sleep_interval=.5)
320 assert task.get_exit_code() is None
321 task.wait(timeout=0)
322
323 # Test timeout for Posix Group fixup task (runs too fast)
324 """
325 groups = PosixGroups(inst, DEFAULT_SUFFIX)
326 start_range = 10000
327 for idx in range(1, 10):
328 group_props = {
329 'cn': 'test_posix_group_' + str(idx),
330 'objectclass': ['posixGroup', 'groupofuniquenames'],
331 'gidNumber': str(idx)
332 }
333 group = groups.create(properties=group_props)
334 for user_idx in range(start_range, start_range + 1000):
335 entry_idx = str(user_idx).zfill(6)
336 dn = f"uid=user{entry_idx},ou=people,{DEFAULT_SUFFIX}"
337 group.add('memberuid', dn)
338 group.add('uniquemember', dn)
339 start_range += 1000
340
341 task = MemberUidFixupTask(inst).create(properties={
342 'basedn': DEFAULT_SUFFIX,
343 'filter': "objectClass=*"
344 })
345 task.wait(timeout=.5, sleep_interval=.5)
346 assert task.get_exit_code() is None
347 task.wait(timeout=0)
348 """
349
350 # Test timeout for memberOf fixup task
351 groups = Groups(inst, DEFAULT_SUFFIX)
352 group_props = {'cn': 'test_group'}
353 group = groups.create(properties=group_props)
354 for idx in range(5000, 6000):
355 entry_idx = str(idx).zfill(6)
356 dn = f"uid=user{entry_idx},ou=people,{DEFAULT_SUFFIX}"
357 group.add_member(dn)
358
359 task = MemberOfFixupTask(inst).create(properties={
360 'basedn': DEFAULT_SUFFIX,
361 'filter': "objectClass=*"
362 })
363 task.wait(timeout=.5, sleep_interval=.5)
364 assert task.get_exit_code() is None
365 task.wait(timeout=0)
366
367 # Test timeout for entryuuid fixup task
368 task = EntryUUIDFixupTask(inst).create(properties={
369 'basedn': DEFAULT_SUFFIX,
370 'filter': "objectClass=*"
371 })
372 task.wait(timeout=.5, sleep_interval=.5)
373 assert task.get_exit_code() is None
374 task.wait(timeout=0)
375
376 # test timeout for linked attrs fixup (runs too fast)
377 """
378 task = FixupLinkedAttributesTask(inst).create(properties={
379 'basedn': DEFAULT_SUFFIX,
380 'filter': "objectClass=*"
381 })
382 task.wait(timeout=.5, sleep_interval=.5)
383 assert task.get_exit_code() is None
384 task.wait(timeout=0)
385 """
386
387 # Test time out for db compact task (runs too fast)
388 """
389 task = DBCompactTask(inst).create()
390 task.wait(timeout=.5, sleep_interval=.5)
391 assert task.get_exit_code() is None
392 task.wait(timeout=0)
393 """
394
395
396 if __name__ == '__main__':
397 # Run isolated
398 # -s for DEBUG mode
399 CURRENT_FILE = os.path.realpath(__file__)
400 pytest.main(["-s", CURRENT_FILE])