Debian Codegraph @

dirsrvtests/tests/suites/import/import_test.py

raw
1 # --- BEGIN COPYRIGHT BLOCK ---
2 # Copyright (C) 2023 Red Hat, Inc.
3 # All rights reserved.
4 #
5 # License: GPL (version 3 or any later version).
6 # See LICENSE for details.
7 # --- END COPYRIGHT BLOCK ---
8
9 """
10 Will test Import (Offline/Online)
11 """
12
13 import os
14 import pytest
15 import time
16 import glob
17 import re
18 import logging
19 import subprocess
20 from datetime import datetime
21 from test389.topologies import topology_st as topo
22 from test389.topologies import topology_m2 as topo_m2
23 from lib389._constants import DEFAULT_BENAME, DEFAULT_SUFFIX, TaskWarning
24 from lib389.dbgen import dbgen_users
25 from lib389.tasks import ImportTask
26 from lib389.index import Indexes
27 from lib389.monitor import Monitor
28 from lib389.backend import Backends
29 from lib389.config import LDBMConfig
30 from lib389.config import LMDB_LDBMConfig, BDB_LDBMConfig
31 from lib389.utils import ds_is_newer, get_default_db_lib
32 from lib389.idm.user import UserAccount, UserAccounts
33 from lib389.idm.account import Accounts
34 from lib389.cli_ctl.dbtasks import dbtasks_ldif2db
35 from lib389.cli_base import FakeArgs
36
37 pytestmark = pytest.mark.tier1
38
39 DEBUGGING = os.getenv("DEBUGGING", default=False)
40 if DEBUGGING:
41 logging.getLogger(__name__).setLevel(logging.DEBUG)
42 else:
43 logging.getLogger(__name__).setLevel(logging.INFO)
44 log = logging.getLogger(__name__)
45
46 bdb_values = {
47 'wait30': 30
48 }
49
50 # Note: I still sometime get failure with a 60s timeout so lets use 90s
51 mdb_values = {
52 'wait30': 90
53 }
54
55 if get_default_db_lib() == 'bdb':
56 values = bdb_values
57 else:
58 values = mdb_values
59
60
61 def _generate_ldif(topo, no_no):
62 """
63 Will generate the ldifs
64 """
65 ldif_dir = topo.standalone.get_ldif_dir()
66 import_ldif = ldif_dir + '/basic_import.ldif'
67 if os.path.isfile(import_ldif):
68 pass
69 else:
70 dbgen_users(topo.standalone, no_no, import_ldif, DEFAULT_SUFFIX)
71
72
73 def _check_users_before_test(topo, no_no):
74 """
75 Will check no user before test.
76 """
77 accounts = Accounts(topo.standalone, DEFAULT_SUFFIX)
78 assert len(accounts.filter('(uid=*)')) < no_no
79
80
81 def _search_for_user(topo, no_n0):
82 """
83 Will make sure that users are imported
84 """
85 accounts = Accounts(topo.standalone, DEFAULT_SUFFIX)
86 assert len(accounts.filter('(uid=*)')) == no_n0
87
88
89 def _import_clean_topo(topo):
90 """
91 Cleanup after import
92 """
93 accounts = Accounts(topo.standalone, DEFAULT_SUFFIX)
94 for i in accounts.filter('(uid=*)'):
95 UserAccount(topo.standalone, i.dn).delete()
96
97 ldif_dir = topo.standalone.get_ldif_dir()
98 import_ldif = ldif_dir + '/basic_import.ldif'
99 if os.path.exists(import_ldif):
100 os.remove(import_ldif)
101 syntax_err_ldif = ldif_dir + '/syntax_err.dif'
102 if os.path.exists(syntax_err_ldif):
103 os.remove(syntax_err_ldif)
104
105
106 @pytest.fixture(scope="function")
107 def _import_clean(request, topo):
108 request.addfinalizer(lambda: _import_clean_topo(topo))
109
110
111 def _import_offline(topo, no_no):
112 """
113 Will import ldifs offline
114 """
115 _check_users_before_test(topo, no_no)
116 ldif_dir = topo.standalone.get_ldif_dir()
117 import_ldif = ldif_dir + '/basic_import.ldif'
118 # Generate ldif
119 _generate_ldif(topo, no_no)
120 # Offline import
121 topo.standalone.stop()
122 t1 = time.time()
123 if not topo.standalone.ldif2db('userRoot', None, None, None, import_ldif):
124 topo.standalone.start()
125 assert False
126 total_time = time.time() - t1
127 topo.standalone.start()
128 _search_for_user(topo, no_no)
129 return total_time
130
131
132 def _import_online(topo, no_no):
133 """
134 Will import ldifs online
135 """
136 _check_users_before_test(topo, no_no)
137 ldif_dir = topo.standalone.get_ldif_dir()
138 import_ldif = ldif_dir + '/basic_import.ldif'
139 _generate_ldif(topo, no_no)
140 # Online
141 import_task = ImportTask(topo.standalone)
142 import_task.import_suffix_from_ldif(ldiffile=import_ldif, suffix=DEFAULT_SUFFIX)
143
144 # Wait a bit till the task is created and available for searching
145 if ds_is_newer('1.4.1.2'):
146 for x in range(60):
147 if import_task.present('nstaskcreated'):
148 break
149 time.sleep(0.5)
150 assert import_task.present('nstaskcreated')
151 else:
152 time.sleep(0.5)
153
154 # Good as place as any to quick test the task has some expected attributes
155 assert import_task.present('nstasklog')
156 assert import_task.present('nstaskcurrentitem')
157 assert import_task.present('nstasktotalitems')
158 assert import_task.present('ttl')
159 import_task.wait()
160 topo.standalone.searchAccessLog('ADD dn="cn=import')
161 topo.standalone.searchErrorsLog('import userRoot: Import complete.')
162 _search_for_user(topo, no_no)
163
164
165 def _create_bogus_ldif(topo):
166 """
167 Will create bogus ldifs
168 """
169 ldif_dir = topo.standalone.get_ldif_dir()
170 line1 = r'dn: cn=Eladio \"A\"\, Santabarbara\, (A\, B\, C),ou=Accounting, dc=example,dc=com'
171 line2 = """objectClass: top
172 objectClass: person
173 objectClass: organizationalPerson
174 objectClass: inetOrgPerson
175 cn: Eladio "A", Santabarbara, (A, B, C)
176 cn: Eladio Santabarbara
177 sn: Santabarbara
178 givenName: Eladio
179 ou: Accounting"""
180 with open(f'{ldif_dir}/bogus.dif', 'w') as out:
181 out.write(f'{line1}{line2}')
182 out.close()
183 import_ldif1 = ldif_dir + '/bogus.ldif'
184 return import_ldif1
185
186
187 def _create_syntax_err_ldif(topo):
188 """
189 Create an ldif file, which contains an entry that violates syntax check
190 """
191 ldif_dir = topo.standalone.get_ldif_dir()
192 line1 = """dn: dc=example,dc=com
193 objectClass: top
194 objectClass: domain
195 dc: example
196
197 dn: ou=groups,dc=example,dc=com
198 objectClass: top
199 objectClass: organizationalUnit
200 ou: groups
201
202 dn: uid=JHunt,ou=groups,dc=example,dc=com
203 objectClass: top
204 objectClass: person
205 objectClass: organizationalPerson
206 objectClass: inetOrgPerson
207 objectclass: inetUser
208 cn: James Hunt
209 sn: Hunt
210 uid: JHunt
211 givenName:
212 """
213 with open(f'{ldif_dir}/syntax_err.ldif', 'w') as out:
214 out.write(f'{line1}')
215 os.chmod(out.name, 0o777)
216 out.close()
217 import_ldif1 = ldif_dir + '/syntax_err.ldif'
218 return import_ldif1
219
220
221 def _now():
222 """
223 Get current time with the format that _check_for_core requires
224 """
225 now = datetime.now()
226 return now.strftime("%Y-%m-%d %H:%M:%S")
227
228
229 def __check_for_core(now):
230 """
231 Check if ns-slapd generated a core since the provided date by looking in the system logs.
232 """
233 cmd = [ 'journalctl' ,'-S', now, '-t', 'audit', '-g', 'ANOM_ABEND.*ns-slapd' ]
234 result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
235 if result.returncode != 1:
236 # journalctl returns 1 if there is no matching records, and 0 if there are records
237 log.error('journalctl output is:\n%s' % result.stdout)
238 raise AssertionError(f'journalctl reported that ns-slapd crashes after {now}')
239
240
241 def test_import_with_index(topo, _import_clean):
242 """
243 Add an index, then import via cn=tasks
244
245 :id: 9ddaf0df-7298-42bb-bdfa-a889ee68bc09
246 :setup: Standalone Instance
247 :steps:
248 1. Creating the room number index
249 2. Importing online
250 3. Import is done -- verifying that it worked
251 :expectedresults:
252 1. Operation successful
253 2. Operation successful
254 3. Operation successful
255 """
256 place = topo.standalone.dbdir
257 if topo.standalone.is_dbi_supported():
258 assert not topo.standalone.is_dbi('userRoot/roomNumber.db')
259 else:
260 assert not glob.glob(f'{place}/userRoot/roomNumber.db*', recursive=True)
261 # Creating the room number index
262 indexes = Indexes(topo.standalone)
263 indexes.create(properties={
264 'cn': 'roomNumber',
265 'nsSystemIndex': 'false',
266 'nsIndexType': 'eq'})
267 topo.standalone.restart()
268 # Importing online
269 _import_online(topo, 5)
270 # Import is done -- verifying that it worked
271 if topo.standalone.is_dbi_supported():
272 assert topo.standalone.is_dbi('userRoot/roomNumber.db')
273 else:
274 assert glob.glob(f'{place}/userRoot/roomNumber.db*', recursive=True)
275
276
277 def test_online_import_with_warning(topo, _import_clean):
278 """
279 Import an ldif file with syntax errors, verify skipped entry warning code
280
281 :id: 9b44cd0e-9d4b-4ae9-b750-cc7ba58d4529
282 :setup: Standalone Instance
283 :steps:
284 1. Create standalone Instance
285 2. Create an ldif file with an entry that violates syntax check (empty givenname)
286 3. Online import of troublesome ldif file
287 :expectedresults:
288 1. Successful import with skipped entry warning
289 """
290 topo.standalone.restart()
291
292 import_task = ImportTask(topo.standalone)
293 import_ldif1 = _create_syntax_err_ldif(topo)
294
295 # Importing the offending ldif file - online
296 import_task.import_suffix_from_ldif(ldiffile=import_ldif1, suffix=DEFAULT_SUFFIX)
297
298 # There is just a single entry in this ldif
299 import_task.wait(5)
300
301 # Check for the task nsTaskWarning attr, make sure its set to skipped entry code
302 assert import_task.present('nstaskwarning')
303 assert TaskWarning.WARN_SKIPPED_IMPORT_ENTRY == import_task.get_task_warn()
304
305
306 def test_crash_on_ldif2db(topo, _import_clean):
307 """
308 Delete the cn=monitor entry for an LDBM backend instance. Doing this will
309 cause the DS to re-create that entry the next time it starts up.
310
311 :id: aecad390-9352-11ea-8a31-8c16451d917b
312 :setup: Standalone Instance
313 :steps:
314 1. Delete the cn=monitor entry for an LDBM backend instance
315 2. Restart the server and verify that the LDBM monitor entry was re-created.
316 :expectedresults:
317 1. Operation successful
318 2. Operation successful
319 """
320 # Delete the cn=monitor entry for an LDBM backend instance. Doing this will
321 # cause the DS to re-create that entry the next time it starts up.
322 monitor = Monitor(topo.standalone)
323 monitor.delete()
324 # Restart the server and verify that the LDBM monitor entry was re-created.
325 _import_offline(topo, 5)
326
327
328 def test_ldif2db_allows_entries_without_a_parent_to_be_imported(topo, _import_clean):
329 """Should reject import of entries that's missing parent suffix
330
331 :id: 27195cea-9c0e-11ea-800b-8c16451d917b
332 :setup: Standalone Instance
333 :steps:
334 1. Import the offending LDIF data - offline
335 2. Violates schema, ending line
336 :expectedresults:
337 1. Operation successful
338 2. Operation Fail
339 """
340 import_ldif1 = _create_bogus_ldif(topo)
341 # Import the offending LDIF data - offline
342 topo.standalone.stop()
343 topo.standalone.ldif2db('userRoot', None, None, None, import_ldif1)
344 # which violates schema, ending line
345 topo.standalone.searchErrorsLog('import_producer - import userRoot: Skipping entry '
346 '"dc=example,dc=com" which violates schema')
347 topo.standalone.start()
348
349
350 def test_ldif2db_syntax_check(topo, _import_clean):
351 """ldif2db should return a warning when a skipped entry has occured.
352
353 :id: 85e75670-42c5-4062-9edc-7f117c97a06f
354 :setup:
355 1. Standalone Instance
356 2. Ldif entry that violates syntax check rule (empty givenname)
357 :steps:
358 1. Create an ldif file which violates the syntax checking rule
359 2. Stop the server and import ldif file with ldif2db
360 :expectedresults:
361 1. ldif2db import returns a warning to signify skipped entries
362 """
363 import_ldif1 = _create_syntax_err_ldif(topo)
364 # Import the offending LDIF data - offline
365 topo.standalone.stop()
366 ret = topo.standalone.ldif2db('userRoot', None, None, None, import_ldif1)
367 assert ret == TaskWarning.WARN_SKIPPED_IMPORT_ENTRY
368 topo.standalone.start()
369
370
371 def test_issue_a_warning_if_the_cache_size_is_smaller(topo, _import_clean):
372 """Report during startup if nsslapd-cachememsize is too small
373
374 :id: 1aa8cbda-9c0e-11ea-9297-8c16451d917b
375 :setup: Standalone Instance
376 :steps:
377 1. Set nsslapd-cache-autosize to 0
378 2. Change cachememsize
379 3. Check that cachememsize is sufficiently small
380 4. Import some users to make id2entry.db big
381 5. Warning message should be there in error logs
382 :expectedresults:
383 1. Operation successful
384 2. Operation successful
385 3. Operation successful
386 4. Operation successful
387 5. Operation successful
388 """
389 if get_default_db_lib() == "bdb":
390 config = BDB_LDBMConfig(topo.standalone)
391 else:
392 config = LMDB_LDBMConfig(topo.standalone)
393 backend = Backends(topo.standalone).list()[0]
394 # Set nsslapd-cache-autosize to 0
395 config.replace('nsslapd-cache-autosize', '0')
396 # Change cachememsize
397 backend.replace('nsslapd-cachememsize', '1')
398 # Check that cachememsize is sufficiently small
399 assert int(backend.get_attr_val_utf8('nsslapd-cachememsize')) < 1500000
400 # Import some users to make id2entry.db big
401 _import_offline(topo, 20)
402 # warning message should look like
403 assert topo.standalone.searchErrorsLog('INFO - ldbm_instance_config_cachememsize_set - '
404 'force a minimal value 512000')
405
406
407 def test_realloc_on_offline_import_export(topo):
408 """Verify offline export/import succeeds with huge entry data and minimal cache settings.
409
410 :id: 2857cfca-0f29-4ea1-b6c3-89d63a0c32f9
411 :setup: Standalone Instance
412 :steps:
413 1. Increase nsslapd-maxbersize and restart
414 2. Create a user with a very large sn value
415 3. Add extensibleObject and a multivalued padding attribute to that user
416 4. Set nsslapd-cache-autosize to 0 and nsslapd-cachememsize to 1
417 5. Stop server, run db2ldif, then run ldif2db on the exported file
418 6. Start server and verify the user still exists
419 :expectedresults:
420 1. Success
421 2. Success
422 3. Success
423 4. Success
424 5. Success
425 6. Success
426 """
427 inst = topo.standalone
428 users = UserAccounts(inst, DEFAULT_SUFFIX)
429 if get_default_db_lib() == "bdb":
430 config = BDB_LDBMConfig(topo.standalone)
431 else:
432 config = LMDB_LDBMConfig(topo.standalone)
433
434 backend = Backends(inst).get(DEFAULT_BENAME)
435
436 uid = 'user1'
437 ldif_path = os.path.join(inst.get_ldif_dir(), f'{inst.serverid}.ldif')
438
439 orig_maxbersize = inst.config.get_attr_val_utf8('nsslapd-maxbersize')
440 orig_cachememsize = backend.get_attr_val_utf8('nsslapd-cachememsize')
441 orig_cacheautosize = config.get_attr_val_utf8('nsslapd-cache-autosize')
442
443 large_sn = 'A' * 10000000
444 padding_vals = [str(n) for n in range(400)]
445
446 try:
447 inst.config.set('nsslapd-maxbersize', '200000000')
448 inst.restart()
449
450 user1 = users.create(properties={
451 'uid': uid,
452 'cn': uid,
453 'sn': large_sn,
454 'uidNumber': '1001',
455 'gidNumber': '1001',
456 'homeDirectory': f'/home/{uid}',
457 'description': 'large entry user',
458 })
459 user1.add('objectClass', 'extensibleObject')
460 user1.add('padding', padding_vals)
461
462 config.replace('nsslapd-cache-autosize', '0')
463 backend.replace('nsslapd-cachememsize', '1')
464
465 inst.stop()
466 assert inst.db2ldif(
467 bename=DEFAULT_BENAME,
468 suffixes=[DEFAULT_SUFFIX],
469 excludeSuffixes=[],
470 encrypt=False,
471 repl_data=True,
472 outputfile=ldif_path,
473 )
474 assert inst.ldif2db(DEFAULT_BENAME, None, None, False, ldif_path)
475 inst.start()
476
477 users._objectclasses.append('extensibleObject')
478 users.get(uid)
479
480 finally:
481 try:
482 users.get(uid).delete()
483 except Exception:
484 pass
485
486 if orig_cachememsize:
487 backend.replace('nsslapd-cachememsize', orig_cachememsize)
488 else:
489 backend.replace('nsslapd-cachememsize', '0')
490
491 if orig_maxbersize is not None:
492 inst.config.set('nsslapd-maxbersize', orig_maxbersize)
493 else:
494 inst.config.set('nsslapd-maxbersize', '0')
495
496 if orig_cacheautosize:
497 config.replace('nsslapd-cache-autosize', orig_cacheautosize)
498 else:
499 config.replace('nsslapd-cache-autosize', '0')
500
501 if os.path.exists(ldif_path):
502 os.remove(ldif_path)
503
504 inst.restart()
505
506
507 @pytest.fixture(scope="function")
508 def _toggle_private_import_mem(request, topo):
509 config = LDBMConfig(topo.standalone)
510 config.replace_many(
511 ('nsslapd-db-private-import-mem', 'on'),
512 ('nsslapd-import-cache-autosize', '0'))
513
514 def finofaci():
515 # nsslapd-import-cache-autosize: off and
516 # nsslapd-db-private-import-mem: off
517 config.replace_many(
518 ('nsslapd-db-private-import-mem', 'off'))
519 request.addfinalizer(finofaci)
520
521
522 #unstable or unstatus tests, skipped for now
523 @pytest.mark.flaky(max_runs=2, min_passes=1)
524 @pytest.mark.skipif(get_default_db_lib() == "mdb", reason="nsslapd-db-private-import-mem and nsslapd-import-cache-autosize parameters are ignored when usign lmdb")
525 def test_fast_slow_import(topo, _toggle_private_import_mem, _import_clean):
526 """With nsslapd-db-private-import-mem: on is faster import.
527
528 :id: 3044331c-9c0e-11ea-ac9f-8c16451d917b
529 :setup: Standalone Instance
530 :steps:
531 1. Let's set nsslapd-db-private-import-mem:on, nsslapd-import-cache-autosize: 0
532 2. Measure offline import time duration total_time1
533 3. Now nsslapd-db-private-import-mem:off
534 4. Measure offline import time duration total_time2
535 5. total_time1 < total_time2
536 6. Set nsslapd-db-private-import-mem:on, nsslapd-import-cache-autosize: -1
537 7. Measure offline import time duration total_time1
538 8. Now nsslapd-db-private-import-mem:off
539 9. Measure offline import time duration total_time2
540 10. total_time1 < total_time2
541 :expectedresults:
542 1. Operation successful
543 2. Operation successful
544 3. Operation successful
545 4. Operation successful
546 5. Operation successful
547 6. Operation successful
548 7. Operation successful
549 8. Operation successful
550 9. Operation successful
551 10. Operation successful
552 """
553 # Let's set nsslapd-db-private-import-mem:on, nsslapd-import-cache-autosize: 0
554 config = LDBMConfig(topo.standalone)
555 # Measure offline import time duration total_time1
556 total_time1 = _import_offline(topo, 1000)
557 # Now nsslapd-db-private-import-mem:off
558 config.replace('nsslapd-db-private-import-mem', 'off')
559 accounts = Accounts(topo.standalone, DEFAULT_SUFFIX)
560 for i in accounts.filter('(uid=*)'):
561 UserAccount(topo.standalone, i.dn).delete()
562 # Measure offline import time duration total_time2
563 total_time2 = _import_offline(topo, 1000)
564 # total_time1 < total_time2
565 log.info("total_time1 = %f" % total_time1)
566 log.info("total_time2 = %f" % total_time2)
567 # Comparing small time values is instable ==> Lets ignore it
568 # assert total_time1 < total_time2
569
570 # Set nsslapd-db-private-import-mem:on, nsslapd-import-cache-autosize: -1
571 config.replace_many(
572 ('nsslapd-db-private-import-mem', 'on'),
573 ('nsslapd-import-cache-autosize', '-1'))
574 for i in accounts.filter('(uid=*)'):
575 UserAccount(topo.standalone, i.dn).delete()
576 # Measure offline import time duration total_time1
577 total_time1 = _import_offline(topo, 1000)
578 # Now nsslapd-db-private-import-mem:off
579 config.replace('nsslapd-db-private-import-mem', 'off')
580 for i in accounts.filter('(uid=*)'):
581 UserAccount(topo.standalone, i.dn).delete()
582 # Measure offline import time duration total_time2
583 total_time2 = _import_offline(topo, 1000)
584 # total_time1 < total_time2
585 log.info("total_time1 = %f" % total_time1)
586 log.info("total_time2 = %f" % total_time2)
587 # Comparing small time values is instable ==> Lets ignore it
588 # assert total_time1 < total_time2
589
590
591 def test_entry_with_escaped_characters_fails_to_import_and_index(topo, _import_clean):
592 """If missing entry_id is found, skip it and continue reading the primary db to be re indexed.
593
594 :id: 358c938c-9c0e-11ea-adbc-8c16451d917b
595 :setup: Standalone Instance
596 :steps:
597 1. Import the example data from ldif.
598 2. Remove some of the other entries that were successfully imported.
599 3. Now re-index the database.
600 4. Should not return error.
601 :expectedresults:
602 1. Operation successful
603 2. Operation successful
604 3. Operation successful
605 4. Operation successful
606 """
607 # Import the example data from ldif
608 _import_offline(topo, 10)
609 count = 0
610 # Remove some of the other entries that were successfully imported
611 for user1 in [user for user in Accounts(topo.standalone, DEFAULT_SUFFIX).list() if user.dn.startswith('uid')]:
612 if count <= 2:
613 UserAccount(topo.standalone, user1.dn).delete()
614 count += 1
615 # Now re-index the database
616 topo.standalone.stop()
617 topo.standalone.db2index(bename="userroot")
618 topo.standalone.start()
619 # Should not return error.
620 assert not topo.standalone.searchErrorsLog('error')
621 assert not topo.standalone.searchErrorsLog('foreman fifo error')
622
623
624 def test_import_perf_after_failure(topo):
625 """Make an import fail by specifying the wrong LDIF file name, then
626 try the import with the correct name. Make sure the import performance
627 is what we expect.
628
629 :id: d21dc67f-475e-402a-be9e-3eeb9181c156
630 :setup: Standalone Instance
631 :steps:
632 1. Build LDIF file
633 2. Import invalid LDIF filename
634 3. Import valid LDIF filename
635 4. Import completes in a timely manner
636 :expectedresults:
637 1. Success
638 2. Success
639 3. Success
640 4. Success
641 """
642
643 ldif_dir = topo.standalone.get_ldif_dir()
644 import_ldif = ldif_dir + '/perf_import.ldif'
645 bad_import_ldif = ldif_dir + '/perf_import_typo.ldif'
646
647 # Build LDIF file
648 dbgen_users(topo.standalone, 30000, import_ldif, DEFAULT_SUFFIX)
649
650 # Online import which fails
651 import_task = ImportTask(topo.standalone)
652 import_task.import_suffix_from_ldif(ldiffile=bad_import_ldif, suffix=DEFAULT_SUFFIX)
653 import_task.wait()
654
655 # Valid online import
656 time.sleep(1)
657 import_task = ImportTask(topo.standalone)
658 import_task.import_suffix_from_ldif(ldiffile=import_ldif, suffix=DEFAULT_SUFFIX)
659 import_task.wait(values['wait30']) # If things go wrong import takes a lot longer than this
660 assert import_task.is_complete()
661
662 # Restart server
663 topo.standalone.restart()
664
665
666 def test_import_wrong_file_path(topo):
667 """Make an import fail by specifying the wrong LDIF file name
668
669 :id: 6795a3cd-b95e-4777-bc77-25ab864882a3
670 :setup: Standalone Instance
671 :steps:
672 1. Do an import with an invalid file path
673 2. Appropriate error is returned
674 :expectedresults:
675 1. Success
676 2. Success
677 """
678 import_ldif = '/nope/perf_import.ldif'
679 args = FakeArgs()
680 args.instance = topo.standalone.serverid
681 args.backend = "userroot"
682 args.encrypted = False
683 args.replication = False
684 args.ldif = import_ldif
685
686 with pytest.raises(ValueError) as e:
687 dbtasks_ldif2db(topo.standalone, log, args)
688 assert "The LDIF file does not exist" in str(e.value)
689
690
691 @pytest.mark.skipif(get_default_db_lib() != "mdb", reason="lmdb specific test")
692 def test_crash_on_ldif2db_with_lmdb(topo, _import_clean):
693 """Make an import fail by specifying a too small db size then check that
694 there is no crash.
695
696 :id: d42585b6-31d0-11ee-8724-482ae39447e5
697 :setup: Standalone Instance
698 :steps:
699 1. Configure a small database size
700 2. Import an ldif with 1K users
701 3. Check that ns-slapd has not aborted
702 4. Import an ldif with 500 users
703 5. Check that ns-slapd has not aborted
704 :expectedresults:
705 1. Success
706 2. Success
707 3. Success (ns-slapd should not have aborted)
708 4. Import should fail
709 5. Success (ns-slapd should not have aborted)
710
711 """
712 TINY_MAP_SIZE = 16 * 1024 * 1024
713 inst = topo.standalone
714 handler = LMDB_LDBMConfig(inst)
715 mapsize = TINY_MAP_SIZE
716 log.info(f'Set lmdb map size to {mapsize}.')
717 handler.replace('nsslapd-mdb-max-size', str(mapsize))
718 inst.stop()
719 for dbfile in ['data.mdb', 'INFO.mdb', 'lock.mdb']:
720 try:
721 os.remove(f'{inst.dbdir}/{dbfile}')
722 except FileNotFoundError:
723 pass
724 inst.start()
725 now = _now()
726 _import_offline(topo, 1000)
727 __check_for_core(now)
728 _import_clean_topo(topo)
729 with pytest.raises(AssertionError):
730 _import_offline(topo, 500_000)
731 __check_for_core(now)
732
733 def test_online_import_under_load(topo):
734 """Perform an online import while the server is under load
735
736 :id: 56f7ac6f-4285-4bc4-8822-5ae9b502eee3
737 :setup: Standalone Instance
738 :steps:
739 1. Create and import LDIF for ldclt load
740 2. Start ldclt
741 3. Start online import
742 4. Wait a bit and kill the ldclt load
743 5. Check import task successfully completed
744
745 :expectedresults:
746 1. Success
747 2. Success
748 3. Success
749 4. Success
750 5. Success
751 """
752 inst = topo.standalone
753
754 # Create and import ldif to use with ldclt
755 ldif_dir = inst.get_ldif_dir()
756 import_ldif = ldif_dir + '/stress_import.ldif'
757 dbgen_users(inst, 1000, import_ldif, generic=True, suffix=DEFAULT_SUFFIX)
758 import_task = ImportTask(topo.standalone)
759 import_task.import_suffix_from_ldif(ldiffile=import_ldif,
760 suffix=DEFAULT_SUFFIX)
761 import_task.wait()
762 assert import_task.get_exit_code() == 0
763
764 # Start ldclt load
765 ldclt_cmd = [
766 '%s/ldclt' % inst.get_bin_dir(),
767 '-h', inst.host, '-p', str(inst.port),
768 '-f', '(uid=userXXXX)', '-e', 'esearch,random',
769 '-r1', '-R999', '-Q'
770 ]
771 p = subprocess.Popen(ldclt_cmd, start_new_session=True,
772 stdout=subprocess.PIPE)
773 time.sleep(1)
774
775 # Start online import
776 import_task = ImportTask(topo.standalone)
777 import_task.import_suffix_from_ldif(ldiffile=import_ldif,
778 suffix=DEFAULT_SUFFIX)
779
780 # Wait a bit till the task is created and available for searching
781 for x in range(10):
782 if import_task.present('nstaskcreated'):
783 break
784 time.sleep(0.5)
785 assert import_task.present('nstaskcreated')
786
787 # Stop the load
788 time.sleep(3)
789 cmd = ['kill', '-9', str(p.pid)]
790 subprocess.Popen(cmd, stdout=subprocess.PIPE)
791
792 # import should finish
793 import_task.wait()
794 assert import_task.get_exit_code() == 0
795
796
797 def test_duplicate_nsuniqueid(topo_m2, request):
798 """Test that after an offline import all
799 nsuniqueid are different
800
801 :id: a2541677-a288-4633-bacf-4050cc56016d
802 :setup: MMR with 2 suppliers
803 :steps:
804 1. stop the instance to do offline operations
805 2. Generate a 5K users LDIF file
806 3. Check that no uniqueid are present in the generated file
807 4. import the generated LDIF
808 5. export the database
809 6. Check that that exported LDIF contains more than 5K nsuniqueid
810 7. Check that there is no duplicate nsuniqued in exported LDIF
811 :expectedresults:
812 1. Should succeeds
813 2. Should succeeds
814 3. Should succeeds
815 4. Should succeeds
816 5. Should succeeds
817 6. Should succeeds
818 7. Should succeeds
819 """
820 m1 = topo_m2.ms["supplier1"]
821
822 # Stop the instance
823 m1.stop()
824
825 # Generate a test ldif (5k entries)
826 log.info("Generating LDIF...")
827 ldif_dir = m1.get_ldif_dir()
828 import_ldif = ldif_dir + '/5k_users_import.ldif'
829 dbgen_users(m1, 5000, import_ldif, DEFAULT_SUFFIX)
830
831 # Check that the generated LDIF does not contain nsuniqueid
832 all_nsuniqueid = []
833 with open(import_ldif, 'r') as file:
834 for line in file:
835 if line.lower().startswith("nsuniqueid: "):
836 all_nsuniqueid.append(line.split(': ')[1])
837 log.info("import file contains " + str(len(all_nsuniqueid)) + " nsuniqueid")
838 assert len(all_nsuniqueid) == 0
839
840 # Import the "nsuniquied free" LDIF file
841 if not m1.ldif2db('userRoot', None, None, None, import_ldif):
842 assert False
843
844 # Export the DB that now should contain nsuniqueid
845 export_ldif = ldif_dir + '/5k_user_export.ldif'
846 log.info("export to file " + export_ldif)
847 m1.db2ldif(bename=DEFAULT_BENAME, suffixes=[DEFAULT_SUFFIX],
848 excludeSuffixes=None, repl_data=False,
849 outputfile=export_ldif, encrypt=False)
850
851 # Check that the export LDIF contain nsuniqueid
852 all_nsuniqueid = []
853 with open(export_ldif, 'r') as file:
854 for line in file:
855 if line.lower().startswith("nsuniqueid: "):
856 all_nsuniqueid.append(line.split(': ')[1])
857 log.info("export file " + export_ldif + " contains " + str(len(all_nsuniqueid)) + " nsuniqueid")
858 assert len(all_nsuniqueid) >= 5000
859
860 # Check that the nsuniqueid are unique
861 assert len(set(all_nsuniqueid)) == len(all_nsuniqueid)
862
863 def fin():
864 if os.path.exists(import_ldif):
865 os.remove(import_ldif)
866 if os.path.exists(export_ldif):
867 os.remove(export_ldif)
868 m1.start
869
870 request.addfinalizer(fin)
871
872 if __name__ == '__main__':
873 # Run isolated
874 # -s for DEBUG mode
875 CURRENT_FILE = os.path.realpath(__file__)
876 pytest.main("-s %s" % CURRENT_FILE)