Debian Codegraph @

dirsrvtests/tests/suites/import/regression_test.py

raw
1 # Copyright (C) 2020 Red Hat, Inc.
2 # All rights reserved.
3 #
4 # License: GPL (version 3 or any later version).
5 # See LICENSE for details.
6 # --- END COPYRIGHT BLOCK ---
7 #
8 from decimal import *
9 import ldap
10 import logging
11 import os
12 import pytest
13 import threading
14 import time
15 from abc import ABC, abstractmethod
16 from lib389.backend import Backends
17 from lib389.properties import TASK_WAIT
18 from test389.topologies import topology_st as topo
19 from lib389.dbgen import dbgen_users
20 from lib389._constants import DEFAULT_SUFFIX, DEFAULT_BENAME
21 from lib389.tasks import *
22 from lib389.idm.user import UserAccounts, UserAccount
23 from lib389.idm.directorymanager import DirectoryManager
24 from lib389.dbgen import *
25 from lib389.utils import *
26 from lib389.config import LMDB_LDBMConfig
27
28 pytestmark = pytest.mark.tier1
29
30 DEBUGGING = os.getenv("DEBUGGING", default=False)
31 if DEBUGGING:
32 logging.getLogger(__name__).setLevel(logging.DEBUG)
33 else:
34 logging.getLogger(__name__).setLevel(logging.INFO)
35 log = logging.getLogger(__name__)
36
37 TEST_SUFFIX1 = "dc=importest1,dc=com"
38 TEST_BACKEND1 = "importest1"
39 TEST_SUFFIX2 = "dc=importest2,dc=com"
40 TEST_BACKEND2 = "importest2"
41 TEST_DEFAULT_SUFFIX = "dc=default,dc=com"
42 TEST_DEFAULT_NAME = "default"
43
44 BIG_MAP_SIZE = 35 * 1024 * 1024 * 1024
45
46 ######################################################################
47 # Some test cases are destroying the backend. #
48 # They should use preserve (or preserve_r if preserve does fail) #
49 # fixture to restore the backend when the test complete #
50 ######################################################################
51
52 def _check_disk_space():
53 if get_default_db_lib() == "mdb":
54 statvfs = os.statvfs(os.environ.get('PREFIX', "/"))
55 available = statvfs.f_frsize * statvfs.f_bavail
56 return available >= BIG_MAP_SIZE
57 return True
58
59
60 @pytest.fixture(scope="function")
61 def _set_mdb_map_size(request, topo):
62 if get_default_db_lib() == "mdb":
63 handler = LMDB_LDBMConfig(topo.standalone)
64 mapsize = BIG_MAP_SIZE
65 log.info(f'Set lmdb map size to {mapsize}.')
66 handler.replace('nsslapd-mdb-max-size', str(mapsize))
67 topo.standalone.restart()
68
69 class AddDelUsers(threading.Thread):
70 def __init__(self, inst):
71 threading.Thread.__init__(self)
72 self.daemon = True
73 self.inst = inst
74 self._should_stop = False
75 self._ran = False
76
77 def run(self):
78 # Add 1000 entries
79 log.info('Run.')
80 conn = DirectoryManager(self.inst.standalone).bind()
81
82 time.sleep(30)
83 log.info('Adding users.')
84 for i in range(1000):
85 user = UserAccounts(conn, DEFAULT_SUFFIX)
86 users = user.create_test_user(uid=i)
87 users.delete()
88 self._ran = True
89 if self._should_stop:
90 break
91 if not self._should_stop:
92 raise RuntimeError('We finished too soon.')
93 conn.close()
94
95 def stop(self):
96 self._should_stop = True
97
98 def has_started(self):
99 return self._ran
100
101
102 def get_backend_by_name(inst, bename):
103 bes = Backends(inst)
104 bename = bename.lower()
105 be = [ be for be in bes if be.get_attr_val_utf8_l('cn') == bename ]
106 return be[0] if len(be) == 1 else None
107
108
109 class LogHandler():
110 def __init__(self, logfd, patterns):
111 self.logfd = logfd
112 self.patterns = [ p.lower() for p in patterns ]
113 self.pos = logfd.tell()
114 self.last_result = None
115
116 def zero(self):
117 return [0 for _ in range(len(self.patterns))]
118
119 def countCaptures(self):
120 res = self.zero()
121 self.logfd.seek(self.pos)
122 for line in iter(self.logfd.readline, ''):
123 # Ignore autotune messages that may confuse the counts
124 if 'bdb_start_autotune' in line:
125 continue
126 # Ignore LMDB size warnings that may confuse the counts
127 if 'dbmdb_ctx_t_db_max_size_set' in line:
128 continue
129 log.info(f'ERROR LOG line is {line.strip()}')
130 for idx,pattern in enumerate(self.patterns):
131 if pattern in line.lower():
132 res[idx] += 1
133 self.pos = self.logfd.tell()
134 self.last_result = res
135 log.info(f'ERROR LOG counts are: {res}')
136 return res
137
138 def seek2end(self):
139 self.pos = os.fstat(self.logfd.fileno()).st_size
140
141 def check(self, idx, val):
142 count = self.last_result[idx]
143 assert count == val , f"Should have {val} '{self.patterns[idx]}' messages but got: {count} - idx = {idx}"
144
145
146 class IEHandler(ABC):
147 def __init__(self, inst, errlog, ldifname, bename=DEFAULT_BENAME, suffix=None):
148 self.inst = inst
149 self.errlog = errlog
150 self.ldifname = ldifname
151 self.bename = bename
152 self.suffix = suffix
153 self.ldif = ldifname if ldifname.startswith('/') else f'{inst.get_ldif_dir()}/{ldifname}.ldif'
154
155 @abstractmethod
156 def get_name(self):
157 pass
158
159 @abstractmethod
160 def _run_task_b(self):
161 pass
162
163 @abstractmethod
164 def _run_task_s(self):
165 pass
166
167 @abstractmethod
168 def _run_offline(self):
169 pass
170
171 @abstractmethod
172 def _set_log_pattern(self, success):
173 pass
174
175 def run(self, extra_checks, success=True):
176 if self.errlog:
177 self._set_log_pattern(success)
178 self.errlog.seek2end()
179
180 if self.inst.status():
181 if self.bename:
182 log.info(f"Performing online {self.get_name()} of backend {self.bename} into LDIF file {self.ldif}")
183 r = self._run_task_b()
184 else:
185 log.info(f"Performing online {self.get_name()} of suffix {self.suffix} into LDIF file {self.ldif}")
186 r = self._run_task_s()
187 r.wait()
188 time.sleep(1)
189 else:
190 if self.bename:
191 log.info(f"Performing offline {self.get_name()} of backend {self.bename} into LDIF file {self.ldif}")
192 else:
193 log.info(f"Performing offline {self.get_name()} of suffix {self.suffix} into LDIF file {self.ldif}")
194 self._run_offline()
195 if self.errlog:
196 expected_counts = ['*' for _ in range(len(self.errlog.patterns))]
197 for (idx, val) in extra_checks:
198 expected_counts[idx] = val
199 res = self.errlog.countCaptures()
200 log.info(f'Expected errorlog counts are: {expected_counts}')
201 if success is True or success is False:
202 log.info(f'Number of {self.errlog.patterns[0]} in errorlog is: {res[0]}')
203 assert res[0] >= 1
204 for (idx, val) in extra_checks:
205 self.errlog.check(idx, val)
206
207 def check_db(self):
208 assert self.inst.dbscan(bename=self.bename, index='id2entry')
209
210
211 class Importer(IEHandler):
212 def get_name(self):
213 return "import"
214
215 def _set_log_pattern(self, success):
216 if success is True:
217 self.errlog.patterns[0] = 'import complete'
218 elif success is False:
219 self.errlog.patterns[0] = 'import failed'
220
221 def _run_task_b(self):
222 bes = Backends(self.inst)
223 r = bes.import_ldif(self.bename, [self.ldif,], include_suffixes=self.suffix)
224 return r
225
226 def _run_task_s(self):
227 r = ImportTask(self.inst)
228 r.import_suffix_from_ldif(self.ldif, self.suffix)
229 return r
230
231 def _run_offline(self):
232 log.info(f'self.inst.ldif2db({self.bename}, {self.suffix}, ...)')
233 if self.suffix is None:
234 self.inst.ldif2db(self.bename, self.suffix, None, False, self.ldif)
235 else:
236 self.inst.ldif2db(self.bename, [self.suffix, ], None, False, self.ldif)
237
238
239 class Exporter(IEHandler):
240 def get_name(self):
241 return "export"
242
243 def _set_log_pattern(self, success):
244 if success is True:
245 self.errlog.patterns[0] = 'export finished'
246 elif success is False:
247 self.errlog.patterns[0] = 'export failed'
248
249 def _run_task_b(self):
250 bes = Backends(self.inst)
251 r = bes.export_ldif(self.bename, self.ldif, include_suffixes=self.suffix)
252 return r
253
254 def _run_task_s(self):
255 r = ExportTask(self.inst)
256 r.export_suffix_to_ldif(self.ldif, self.suffix)
257 return r
258
259 def _run_offline(self):
260 self.inst.db2ldif(self.bename, self.suffix, None, False, False, self.ldif)
261
262
263 def preserve_func(topo, request, restart):
264 # Ensure that topology get preserved helper
265 inst = topo.standalone
266
267 def fin():
268 if restart:
269 inst.restart()
270 Importer(inst, None, "save").run(())
271
272 r = Exporter(inst, None, "save")
273 if not os.path.isfile(r.ldif):
274 r.run(())
275 request.addfinalizer(fin)
276
277
278 @pytest.fixture(scope="function")
279 def preserve(topo, request):
280 # Ensure that topology get preserved (no restart)
281 preserve_func(topo, request, False)
282
283
284 @pytest.fixture(scope="function")
285 def preserve_r(topo, request):
286 # Ensure that topology get preserved (with restart)
287 preserve_func(topo, request, True)
288
289
290 @pytest.fixture(scope="function")
291 def verify(topo):
292 # Check that backend is not broken
293 inst = topo.standalone
294 dn=f'uid=demo_user,ou=people,{DEFAULT_SUFFIX}'
295 assert UserAccount(inst,dn).exists()
296
297
298 def test_replay_import_operation(topo, preserve_r, verify):
299 """ Check after certain failed import operation, is it
300 possible to replay an import operation
301
302 :id: 5f5ca532-8e18-4f7b-86bc-ac585215a473
303 :feature: Import
304 :setup: Standalone instance
305 :steps:
306 1. Export the backend into an ldif file
307 2. Perform high load of operation on the server (Add/Del users)
308 3. Perform an import operation
309 4. Again perform an import operation (same as 3)
310 :expectedresults:
311 1. It should be successful
312 2. It should be successful
313 3. It should be unsuccessful, should give OPERATIONS_ERROR
314 4. It should be successful now
315 """
316 log.info("Exporting LDIF online...")
317 ldif_dir = topo.standalone.get_ldif_dir()
318 export_ldif = ldif_dir + '/export.ldif'
319
320 r = ExportTask(topo.standalone)
321 r.export_suffix_to_ldif(ldiffile=export_ldif, suffix=DEFAULT_SUFFIX)
322 r.wait()
323 add_del_users1 = AddDelUsers(topo)
324 add_del_users1.start()
325
326 log.info("Importing LDIF online, should raise operation error.")
327
328 trials = 0
329 while not add_del_users1.has_started() and trials < 10:
330 trials += 1
331 time.sleep(1)
332 r = ImportTask(topo.standalone)
333 try:
334 r.import_suffix_from_ldif(ldiffile=export_ldif, suffix=DEFAULT_SUFFIX)
335 except ldap.OPERATIONS_ERROR:
336 break
337 log.info(f'Looping. Tried {trials} times so far.')
338 add_del_users1.stop()
339 add_del_users1.join()
340
341 log.info("Importing LDIF online")
342
343 r = ImportTask(topo.standalone)
344 r.import_suffix_from_ldif(ldiffile=export_ldif, suffix=DEFAULT_SUFFIX)
345
346
347 def test_import_be_default(topo, verify):
348 """ Create a backend using the name "default". previously this name was
349 used int
350
351 :id: 8e507beb-e917-4330-8cac-1ff0eee10508
352 :feature: Import
353 :setup: Standalone instance
354 :steps:
355 1. Create a test suffix using the be name of "default"
356 2. Create an ldif for the "default" backend
357 3. Import ldif
358 4. Verify all entries were imported
359 :expectedresults:
360 1. Success
361 2. Success
362 3. Success
363 4. Success
364 """
365 log.info('Adding suffix:{} and backend: {}...'.format(TEST_DEFAULT_SUFFIX,
366 TEST_DEFAULT_NAME))
367 backends = Backends(topo.standalone)
368 backends.create(properties={'nsslapd-suffix': TEST_DEFAULT_SUFFIX,
369 'name': TEST_DEFAULT_NAME})
370
371 log.info('Create LDIF file and import it...')
372 ldif_dir = topo.standalone.get_ldif_dir()
373 ldif_file = os.path.join(ldif_dir, 'default.ldif')
374 dbgen_users(topo.standalone, 5, ldif_file, TEST_DEFAULT_SUFFIX)
375
376 log.info('Stopping the server and running offline import...')
377 topo.standalone.stop()
378 assert topo.standalone.ldif2db(TEST_DEFAULT_NAME, None, None,
379 None, ldif_file)
380 topo.standalone.start()
381
382 log.info('Verifying entry count after import...')
383 entries = topo.standalone.search_s(TEST_DEFAULT_SUFFIX,
384 ldap.SCOPE_SUBTREE,
385 "(objectclass=*)")
386 assert len(entries) > 1
387
388 log.info('Test PASSED')
389
390
391 def test_del_suffix_import(topo, verify):
392 """Adding a database entry fails if the same database was deleted after an import
393
394 :id: 652421ef-738b-47ed-80ec-2ceece6b5d77
395 :feature: Import
396 :setup: Standalone instance
397 :steps: 1. Create a test suffix and add few entries
398 2. Stop the server and do offline import using ldif2db
399 3. Delete the suffix backend
400 4. Add a new suffix with the same database name
401 5. Check if adding the same database name is a success
402 :expectedresults: Adding database with the same name should be successful
403 """
404
405 log.info('Adding suffix:{} and backend: {}'.format(TEST_SUFFIX1, TEST_BACKEND1))
406 backends = Backends(topo.standalone)
407 backend = backends.create(properties={'nsslapd-suffix': TEST_SUFFIX1,
408 'name': TEST_BACKEND1})
409
410 log.info('Create LDIF file and import it')
411 ldif_dir = topo.standalone.get_ldif_dir()
412 ldif_file = os.path.join(ldif_dir, 'suffix_del1.ldif')
413
414 dbgen_users(topo.standalone, 10, ldif_file, TEST_SUFFIX1)
415
416 log.info('Stopping the server and running offline import')
417 topo.standalone.stop()
418 assert topo.standalone.ldif2db(TEST_BACKEND1, TEST_SUFFIX1, None, None, ldif_file)
419 topo.standalone.start()
420
421 log.info('Deleting suffix-{}'.format(TEST_SUFFIX2))
422 backend.delete()
423
424 log.info('Adding the same database-{} after deleting it'.format(TEST_BACKEND1))
425 backends.create(properties={'nsslapd-suffix': TEST_SUFFIX1,
426 'name': TEST_BACKEND1})
427
428
429 def test_del_suffix_backend(topo, verify):
430 """Adding a database entry fails if the same database was deleted after an import
431
432 :id: ac702c35-74b6-434e-8e30-316433f3e91a
433 :feature: Import
434 :setup: Standalone instance
435 :steps: 1. Create a test suffix and add entries
436 2. Stop the server and do online import using ldif2db
437 3. Delete the suffix backend
438 4. Add a new suffix with the same database name
439 5. Restart the server and check the status
440 :expectedresults: Adding database with the same name should be successful and the server should not hang
441 """
442
443 log.info('Adding suffix:{} and backend: {}'.format(TEST_SUFFIX2, TEST_BACKEND2))
444 backends = Backends(topo.standalone)
445 backend = backends.create(properties={'nsslapd-suffix': TEST_SUFFIX2,
446 'name': TEST_BACKEND2})
447
448 log.info('Create LDIF file and import it')
449 ldif_dir = topo.standalone.get_ldif_dir()
450 ldif_file = os.path.join(ldif_dir, 'suffix_del2.ldif')
451
452 dbgen_users(topo.standalone, 10, ldif_file, TEST_SUFFIX2)
453
454 topo.standalone.tasks.importLDIF(suffix=TEST_SUFFIX2, input_file=ldif_file, args={TASK_WAIT: True})
455
456 log.info('Deleting suffix-{}'.format(TEST_SUFFIX2))
457 backend.delete()
458
459 log.info('Adding the same database-{} after deleting it'.format(TEST_BACKEND2))
460 backends.create(properties={'nsslapd-suffix': TEST_SUFFIX2,
461 'name': TEST_BACKEND2})
462 log.info('Checking if server can be restarted after re-adding the same database')
463 topo.standalone.restart()
464 assert not topo.standalone.detectDisorderlyShutdown()
465
466
467 def test_import_duplicate_dn(topo, preserve, verify):
468 """Import ldif with duplicate DNs, should not log error "unable to flush"
469
470 :id: dce2b898-119d-42b8-a236-1130f58bff17
471 :setup: Standalone instance, ldif file with duplicate entries
472 :steps:
473 1. Create a ldif file with duplicate entries
474 2. Import ldif file to DS
475 3. Check error log file, it should not log "unable to flush"
476 4. Check error log file, it should log "Duplicated DN detected"
477 :expectedresults:
478 1. Success
479 2. Success
480 3. Success
481 4. Success
482 """
483
484 standalone = topo.standalone
485
486 log.info('Delete the previous error logs')
487 standalone.deleteErrorLogs()
488
489 log.info('Create import file')
490 l = """dn: dc=example,dc=com
491 objectclass: top
492 objectclass: domain
493 dc: example
494
495 dn: ou=myDups00001,dc=example,dc=com
496 objectclass: top
497 objectclass: organizationalUnit
498 ou: myDups00001
499
500 dn: ou=myDups00001,dc=example,dc=com
501 objectclass: top
502 objectclass: organizationalUnit
503 ou: myDups00001
504 """
505
506 ldif_dir = standalone.get_ldif_dir()
507 ldif_file = os.path.join(ldif_dir, 'data.ldif')
508 with open(ldif_file, "w") as fd:
509 fd.write(l)
510 fd.close()
511 os.chmod(ldif_file, 0o777)
512
513 log.info('Import ldif with duplicate entry')
514 assert standalone.tasks.importLDIF(suffix=DEFAULT_SUFFIX, input_file=ldif_file, args={TASK_WAIT: True})
515
516 log.info('Restart the server to flush the logs')
517 standalone.restart()
518
519 log.info('Error log should not have "unable to flush" message')
520 assert not standalone.ds_error_log.match('.*unable to flush.*')
521
522 log.info('Error log should have "Duplicated DN detected" message')
523 assert standalone.ds_error_log.match('.*Duplicated DN detected.*')
524
525 @pytest.mark.tier2
526 @pytest.mark.skipif(not _check_disk_space(), reason="not enough disk space for lmdb map")
527 @pytest.mark.xfail(ds_is_older("1.3.10.1"), reason="bz1749595 not fixed on versions older than 1.3.10.1")
528 def test_large_ldif2db_ancestorid_index_creation(topo, _set_mdb_map_size, verify):
529 """Import with ldif2db a large file - check that the ancestorid index creation phase has a correct performance
530
531 :id: fe7f78f6-6e60-425d-ad47-b39b67e29113
532 :setup: Standalone instance
533 :steps:
534 1. Delete the previous errors log to start from a fresh one
535 2. Create test suffix and backend
536 3. Create a large nested ldif file
537 4. Stop the server
538 5. Run an offline import
539 6. Restart the server
540 7. Check in the errors log that an independant ancestorid IDs sorting is done
541 8. Get the log of the starting of the ancestorid indexing process
542 9. Get the log of the end of the ancestorid indexing process
543 10. Get the start and end time for ancestorid index creation from these logs
544 11. Calculate the duration of the ancestorid indexing process
545 :expectedresults:
546 1. Success
547 2. Test suffix and backend successfully created
548 3. ldif file successfully created
549 4. Success
550 5. Import is successfully performed
551 6. Success
552 7. Log of ancestorid sorting start and end are present
553 8. Log of the beginning of gathering ancestorid is found
554 9. Log of the final ancestorid index creation is found
555 10. Start and end times are successfully extracted
556 11. The duration of the ancestorid index creation process should be less than 10s
557 """
558
559 ldif_dir = topo.standalone.get_ldif_dir()
560 ldif_file = os.path.join(topo.standalone.ds_paths.ldif_dir, 'large_nested.ldif')
561
562 # Have a reasonable balance between the need for a large ldif file to import and the time of test execution
563 # total number of users
564 num_users = 100000
565
566 # Choose a limited number of users per node to get as much as possible non-leaf entries
567 node_limit = 5
568
569 # top suffix
570 suffix = 'o=test'
571
572 # backend
573 backend = 'test'
574
575 log.info('Delete the previous errors logs')
576 topo.standalone.deleteErrorLogs()
577
578 log.info('Add suffix:{} and backend: {}...'.format(suffix, backend))
579
580 backends = Backends(topo.standalone)
581 backends.create(properties={'nsslapd-suffix': suffix,
582 'name': backend})
583
584 props = {
585 'numUsers' : num_users,
586 'nodeLimit' : node_limit,
587 'suffix' : suffix
588 }
589 instance = topo.standalone
590
591 log.info('Create a large nested ldif file using dbgen : %s' % ldif_file)
592 dbgen_nested_ldif(instance, ldif_file, props)
593
594 log.info('Stop the server and run offline import...')
595 topo.standalone.stop()
596 assert topo.standalone.ldif2db(backend, None, None,
597 None, ldif_file)
598
599 log.info('Starting the server')
600 topo.standalone.start()
601
602 # With lmdb there is no more any special phase for ancestorid
603 # because ancestorsid get updated on the fly while processing the
604 # entryrdn (by up the parents chain to compute the parentid
605 #
606 # But there is still a numSubordinates generation phase
607 if get_default_db_lib() == "mdb":
608 log.info('parse the errors logs to check lines with "Generating numSubordinates complete." are present')
609 end_numsubordinates = str(topo.standalone.ds_error_log.match(r'.*Generating numSubordinates complete.*'))[1:-1]
610 assert len(end_numsubordinates) > 0
611
612 else:
613 log.info('parse the errors logs to check lines with "Starting sort of ancestorid" are present')
614 start_sort_str = str(topo.standalone.ds_error_log.match(r'.*Starting sort of ancestorid non-leaf IDs*'))[1:-1]
615 assert len(start_sort_str) > 0
616
617 log.info('parse the errors logs to check lines with "Finished sort of ancestorid" are present')
618 end_sort_str = str(topo.standalone.ds_error_log.match(r'.*Finished sort of ancestorid non-leaf IDs*'))[1:-1]
619 assert len(end_sort_str) > 0
620
621 log.info('parse the error logs for the line with "Gathering ancestorid non-leaf IDs"')
622 start_ancestorid_indexing_op_str = str(topo.standalone.ds_error_log.match(r'.*Gathering ancestorid non-leaf IDs*'))[1:-1]
623 assert len(start_ancestorid_indexing_op_str) > 0
624
625 log.info('parse the error logs for the line with "Created ancestorid index"')
626 end_ancestorid_indexing_op_str = str(topo.standalone.ds_error_log.match(r'.*Created ancestorid index*'))[1:-1]
627 assert len(end_ancestorid_indexing_op_str) > 0
628
629 log.info('get the ancestorid non-leaf IDs indexing start and end time from the collected strings')
630 # Collected lines look like : '[15/May/2020:05:30:27.245967313 -0400] - INFO - bdb_get_nonleaf_ids - import userRoot: Gathering ancestorid non-leaf IDs...'
631 # We are getting the sec.nanosec part of the date, '27.245967313' in the above example
632 start_time = (start_ancestorid_indexing_op_str.split()[0]).split(':')[3]
633 end_time = (end_ancestorid_indexing_op_str.split()[0]).split(':')[3]
634
635 log.info('Calculate the elapsed time for the ancestorid non-leaf IDs index creation')
636 etime = (Decimal(end_time) - Decimal(start_time))
637 # The time for the ancestorid index creation should be less than 10s for an offline import of an ldif file with 100000 entries / 5 entries per node
638 # Should be adjusted if these numbers are modified in the test
639 assert etime <= 10
640
641
642 def create_backend_and_import(instance, ldif_file, suffix, backend):
643 log.info(f'Add suffix:{suffix} and backend: {backend}...')
644 backends = Backends(instance)
645 backends.create(properties={'nsslapd-suffix': suffix, 'name': backend})
646 props = {'numUsers': 10000, 'nodeLimit': 5, 'suffix': suffix}
647
648 log.info(f'Create a large nested ldif file using dbgen : {ldif_file}')
649 dbgen_nested_ldif(instance, ldif_file, props)
650
651 log.info('Stop the server and run offline import...')
652 instance.stop()
653 log.info('Measure the import time for the ldif file...')
654 start = time.time()
655 assert instance.ldif2db(backend, None, None, None, ldif_file)
656 end = time.time()
657 instance.start()
658 return end - start
659
660
661 @pytest.mark.skipif(get_default_db_lib() == "mdb", reason="Not cache size over mdb")
662 def test_ldif2db_after_backend_create(topo, verify):
663 """Test that ldif2db after backend creation is not slow first time
664
665 :id: c1ab1df7-c70a-46be-bbca-8d65c6ebaa14
666 :setup: Standalone Instance
667 :steps:
668 1. Create backend and suffix
669 2. Generate large LDIF file
670 3. Stop server and run offline import
671 4. Measure import time
672 5. Restart server and repeat steps 1-4 with new backend and suffix
673 :expectedresults:
674 1. Operation successful
675 2. Operation successful
676 3. Operation successful
677 4. Import times should be approximately the same
678 5. Operation successful
679 """
680
681 instance = topo.standalone
682 ldif_dir = instance.get_ldif_dir()
683 ldif_file_1 = os.path.join(ldif_dir, 'large_nested_1.ldif')
684 ldif_file_2 = os.path.join(ldif_dir, 'large_nested_2.ldif')
685
686 import_time_1 = create_backend_and_import(instance, ldif_file_1, 'o=test_1', 'test_1')
687 import_time_2 = create_backend_and_import(instance, ldif_file_2, 'o=test_2', 'test_2')
688
689 log.info('Import times should be approximately the same')
690 assert abs(import_time_1 - import_time_2) < 20
691
692
693 def test_ldif_missing_suffix_entry(topo, request, verify):
694 """Test that ldif2db/import aborts if suffix entry is not in the ldif
695
696 :id: 731bd0d6-8cc8-11f0-8ef2-c85309d5c3e3
697 :setup: Standalone Instance
698 :steps:
699 1. Prepare final cleanup
700 2. Add a few users
701 3. Export ou=people subtree
702 4. Online import using backend name ou=people subtree
703 5. Online import using suffix name ou=people subtree
704 6. Stop the instance
705 7. Offline import using backend name ou=people subtree
706 8. Offline import using suffix name ou=people subtree
707 9. Generate ldif with a far away suffix
708 10. Offline import using backend name and "far" ldif
709 11. Offline import using suffix name and "far" ldif
710 12. Start the instance
711 13. Online import using backend name and "far" ldif
712 14. Online import using suffix name and "far" ldif
713 :expectedresults:
714 1. Operation successful
715 2. Operation successful
716 3. Operation successful
717 4. Import should success, skip all entries, db should exists
718 5. Import should success, skip all entries, db should exists
719 6. Operation successful
720 7. Import should success, skip all entries, db should exists
721 8. Import should success, skip all entries, db should exists
722 9. Operation successful
723 10. Import should success, skip all entries, db should exists
724 11. Import should success, 10 entries skipped, db should exists
725 12. Operation successful
726 13. Import should success, skip all entries, db should exists
727 14. Import should success, 10 entries skipped, db should exists
728 """
729
730 inst = topo.standalone
731 inst.config.set('nsslapd-errorlog-level', '266354688')
732 no_suffix_on = (
733 (1, 0), # no errors are expected.
734 (2, 1), # 1 warning is expected.
735 (3, 0), # no 'no parent' warning is expected.
736 (4, 1), # 1 'all entries were skipped' warning
737 (5, 0), # no 'returning task warning' info message
738 )
739 no_suffix_off = (
740 (1, 0), # no errors are expected.
741 (2, 1), # 1 warning is expected.
742 (3, 0), # no 'no parent' warning is expected.
743 (4, 1), # 1 'all entries were skipped' warning
744 (5, 1), # 1 'returning task warning' info message
745 )
746
747 far_suffix_on = (
748 (1, 0), # no errors are expected.
749 (2, 1), # 1 warning (consolidated, pre-check aborts after 4 entries)
750 (3, 0), # 0 'no parent' warnings (pre-check aborts before processing)
751 (4, 1), # 1 'all entries were skipped' warning (from pre-check)
752 (5, 0), # 0 'returning task warning' info message (online import)
753 )
754 # Backend-specific behavior for orphan detection when suffix parameter is provided
755 nbw = 0 if get_default_db_lib() == "bdb" else 10
756 far_suffix_with_suffix_on = (
757 (1, 0), # no errors are expected.
758 (2, nbw), # 0 (BDB early filtering) or 10 (LMDB orphan detection) warnings
759 (3, nbw), # 0 (BDB early filtering) or 10 (LMDB orphan detection) 'no parent' warnings
760 (4, 0), # 0 'all entries were skipped' warning (no pre-check abort)
761 (5, 0), # 0 'returning task warning' info message (online import)
762 )
763 far_suffix_off = (
764 (1, 0), # no errors are expected.
765 (2, 1), # 1 warning (consolidated, pre-check detects missing suffix)
766 (3, 0), # 0 'no parent' warnings (pre-check aborts before processing)
767 (4, 1), # 1 'all entries were skipped' warning (from pre-check)
768 (5, 1), # 1 'returning task warning' info message (offline import)
769 )
770 far_suffix_with_suffix_off = (
771 (1, 0), # no errors are expected.
772 (2, nbw), # 0 (BDB early filtering) or 10 (LMDB orphan detection) warnings
773 (3, nbw), # 0 (BDB early filtering) or 10 (LMDB orphan detection) 'no parent' warnings
774 (4, 0), # 0 'all entries were skipped' warning (no pre-check abort)
775 (5, 0), # 0 'returning task warning' (rc=0, successful import of suffix)
776 )
777
778 with open(inst.ds_paths.error_log, 'at+') as fd:
779 patterns = (
780 "Reserved for IEHandler",
781 " ERR ",
782 " WARN ",
783 "has no parent",
784 "all entries were skipped",
785 "returning task warning",
786 )
787 errlog = LogHandler(fd, patterns)
788 no_errors = ((1, 0), (2, 0)) # no errors nor warnings are expected.
789
790
791 # 1. Prepare final cleanup
792 Exporter(inst, errlog, "full").run(no_errors)
793
794 def fin():
795 inst.start()
796 with open(inst.ds_paths.error_log, 'at+') as cleanup_fd:
797 cleanup_errlog = LogHandler(cleanup_fd, patterns)
798 Importer(inst, cleanup_errlog, "full").run(no_errors)
799
800 if not DEBUGGING:
801 request.addfinalizer(fin)
802
803 # 2. Add a few users
804 user = UserAccounts(inst, DEFAULT_SUFFIX)
805 users = [ user.create_test_user(uid=i) for i in range(10) ]
806
807 # 3. Export ou=people subtree
808 e = Exporter(inst, errlog, "people", suffix=f'ou=people,{DEFAULT_SUFFIX}')
809 e.run(no_errors) # no errors nor warnings are expected.
810
811 # 4. Online import using backend name ou=people subtree
812 e = Importer(inst, errlog, "people")
813 e.run(no_suffix_on)
814 e.check_db()
815
816 # 5. Online import using suffix name ou=people subtree
817 e = Importer(inst, errlog, "people", suffix=DEFAULT_SUFFIX)
818 e.run(no_suffix_on)
819 e.check_db()
820
821 # 6. Stop the instance
822 inst.stop()
823
824 # 7. Offline import using backend name ou=people subtree
825 e = Importer(inst, errlog, "people")
826 e.run(no_suffix_off)
827 e.check_db()
828
829 # 8. Offline import using suffix name ou=people subtree
830 e = Importer(inst, errlog, "people", suffix=DEFAULT_SUFFIX)
831 e.run(no_suffix_off)
832 e.check_db()
833
834 # 9. Generate ldif with a far away suffix
835 e = Importer(inst, errlog, "full")
836 people_ldif = e.ldif
837 e = Importer(inst, errlog, "far")
838 with open(e.ldif, "wt") as fout:
839 with open(people_ldif, "rt") as fin:
840 # Copy version
841 line = fin.readline()
842 fout.write(line)
843 line = fin.readline()
844 fout.write(line)
845 # Generate fake entries
846 for idx in range(10):
847 fout.write(f"dn: uid=id{idx},dc=foo\nobjectclasses: extensibleObject\n\n")
848 for line in iter(fin.readline, ''):
849 fout.write(line)
850
851 os.chmod(e.ldif, 0o644)
852
853 # 10. Offline import using backend name ou=people subtree
854 e.run(far_suffix_off)
855 e.check_db()
856
857 # 11. Offline import using suffix name ou=people subtree
858 e = Importer(inst, errlog, "far", suffix=DEFAULT_SUFFIX)
859 e.run(far_suffix_with_suffix_off)
860 e.check_db()
861
862 # 12. Start the instance
863 inst.start()
864
865 # 13. Online import using backend name ou=people subtree
866 e = Importer(inst, errlog, "far")
867 e.run(far_suffix_on)
868 e.check_db()
869
870 # 14. Online import using suffix name ou=people subtree
871 e = Importer(inst, errlog, "far", suffix=DEFAULT_SUFFIX)
872 e.run(far_suffix_with_suffix_on)
873 e.check_db()
874
875
876 if __name__ == '__main__':
877 # Run isolated
878 # -s for DEBUG mode
879 CURRENT_FILE = os.path.realpath(__file__)
880 pytest.main("-s {}".format(CURRENT_FILE))