| 1 |
# Copyright (C) 2020 Red Hat, Inc. |
| 2 |
# All rights reserved. |
| 3 |
# |
| 4 |
# License: GPL (version 3 or any later version). |
| 5 |
# See LICENSE for details. |
| 6 |
# --- END COPYRIGHT BLOCK --- |
| 7 |
# |
| 8 |
from decimal import * |
| 9 |
import ldap |
| 10 |
import logging |
| 11 |
import os |
| 12 |
import pytest |
| 13 |
import threading |
| 14 |
import time |
| 15 |
from abc import ABC, abstractmethod |
| 16 |
from lib389.backend import Backends |
| 17 |
from lib389.properties import TASK_WAIT |
| 18 |
from test389.topologies import topology_st as topo |
| 19 |
from lib389.dbgen import dbgen_users |
| 20 |
from lib389._constants import DEFAULT_SUFFIX, DEFAULT_BENAME |
| 21 |
from lib389.tasks import * |
| 22 |
from lib389.idm.user import UserAccounts, UserAccount |
| 23 |
from lib389.idm.directorymanager import DirectoryManager |
| 24 |
from lib389.dbgen import * |
| 25 |
from lib389.utils import * |
| 26 |
from lib389.config import LMDB_LDBMConfig |
| 27 |
|
| 28 |
pytestmark = pytest.mark.tier1 |
| 29 |
|
| 30 |
DEBUGGING = os.getenv("DEBUGGING", default=False) |
| 31 |
if DEBUGGING: |
| 32 |
logging.getLogger(__name__).setLevel(logging.DEBUG) |
| 33 |
else: |
| 34 |
logging.getLogger(__name__).setLevel(logging.INFO) |
| 35 |
log = logging.getLogger(__name__) |
| 36 |
|
| 37 |
TEST_SUFFIX1 = "dc=importest1,dc=com" |
| 38 |
TEST_BACKEND1 = "importest1" |
| 39 |
TEST_SUFFIX2 = "dc=importest2,dc=com" |
| 40 |
TEST_BACKEND2 = "importest2" |
| 41 |
TEST_DEFAULT_SUFFIX = "dc=default,dc=com" |
| 42 |
TEST_DEFAULT_NAME = "default" |
| 43 |
|
| 44 |
BIG_MAP_SIZE = 35 * 1024 * 1024 * 1024 |
| 45 |
|
| 46 |
###################################################################### |
| 47 |
# Some test cases are destroying the backend. # |
| 48 |
# They should use preserve (or preserve_r if preserve does fail) # |
| 49 |
# fixture to restore the backend when the test complete # |
| 50 |
###################################################################### |
| 51 |
|
| 52 |
def _check_disk_space(): |
| 53 |
if get_default_db_lib() == "mdb": |
| 54 |
statvfs = os.statvfs(os.environ.get('PREFIX', "/")) |
| 55 |
available = statvfs.f_frsize * statvfs.f_bavail |
| 56 |
return available >= BIG_MAP_SIZE |
| 57 |
return True |
| 58 |
|
| 59 |
|
| 60 |
@pytest.fixture(scope="function") |
| 61 |
def _set_mdb_map_size(request, topo): |
| 62 |
if get_default_db_lib() == "mdb": |
| 63 |
handler = LMDB_LDBMConfig(topo.standalone) |
| 64 |
mapsize = BIG_MAP_SIZE |
| 65 |
log.info(f'Set lmdb map size to {mapsize}.') |
| 66 |
handler.replace('nsslapd-mdb-max-size', str(mapsize)) |
| 67 |
topo.standalone.restart() |
| 68 |
|
| 69 |
class AddDelUsers(threading.Thread): |
| 70 |
def __init__(self, inst): |
| 71 |
threading.Thread.__init__(self) |
| 72 |
self.daemon = True |
| 73 |
self.inst = inst |
| 74 |
self._should_stop = False |
| 75 |
self._ran = False |
| 76 |
|
| 77 |
def run(self): |
| 78 |
# Add 1000 entries |
| 79 |
log.info('Run.') |
| 80 |
conn = DirectoryManager(self.inst.standalone).bind() |
| 81 |
|
| 82 |
time.sleep(30) |
| 83 |
log.info('Adding users.') |
| 84 |
for i in range(1000): |
| 85 |
user = UserAccounts(conn, DEFAULT_SUFFIX) |
| 86 |
users = user.create_test_user(uid=i) |
| 87 |
users.delete() |
| 88 |
self._ran = True |
| 89 |
if self._should_stop: |
| 90 |
break |
| 91 |
if not self._should_stop: |
| 92 |
raise RuntimeError('We finished too soon.') |
| 93 |
conn.close() |
| 94 |
|
| 95 |
def stop(self): |
| 96 |
self._should_stop = True |
| 97 |
|
| 98 |
def has_started(self): |
| 99 |
return self._ran |
| 100 |
|
| 101 |
|
| 102 |
def get_backend_by_name(inst, bename): |
| 103 |
bes = Backends(inst) |
| 104 |
bename = bename.lower() |
| 105 |
be = [ be for be in bes if be.get_attr_val_utf8_l('cn') == bename ] |
| 106 |
return be[0] if len(be) == 1 else None |
| 107 |
|
| 108 |
|
| 109 |
class LogHandler(): |
| 110 |
def __init__(self, logfd, patterns): |
| 111 |
self.logfd = logfd |
| 112 |
self.patterns = [ p.lower() for p in patterns ] |
| 113 |
self.pos = logfd.tell() |
| 114 |
self.last_result = None |
| 115 |
|
| 116 |
def zero(self): |
| 117 |
return [0 for _ in range(len(self.patterns))] |
| 118 |
|
| 119 |
def countCaptures(self): |
| 120 |
res = self.zero() |
| 121 |
self.logfd.seek(self.pos) |
| 122 |
for line in iter(self.logfd.readline, ''): |
| 123 |
# Ignore autotune messages that may confuse the counts |
| 124 |
if 'bdb_start_autotune' in line: |
| 125 |
continue |
| 126 |
# Ignore LMDB size warnings that may confuse the counts |
| 127 |
if 'dbmdb_ctx_t_db_max_size_set' in line: |
| 128 |
continue |
| 129 |
log.info(f'ERROR LOG line is {line.strip()}') |
| 130 |
for idx,pattern in enumerate(self.patterns): |
| 131 |
if pattern in line.lower(): |
| 132 |
res[idx] += 1 |
| 133 |
self.pos = self.logfd.tell() |
| 134 |
self.last_result = res |
| 135 |
log.info(f'ERROR LOG counts are: {res}') |
| 136 |
return res |
| 137 |
|
| 138 |
def seek2end(self): |
| 139 |
self.pos = os.fstat(self.logfd.fileno()).st_size |
| 140 |
|
| 141 |
def check(self, idx, val): |
| 142 |
count = self.last_result[idx] |
| 143 |
assert count == val , f"Should have {val} '{self.patterns[idx]}' messages but got: {count} - idx = {idx}" |
| 144 |
|
| 145 |
|
| 146 |
class IEHandler(ABC): |
| 147 |
def __init__(self, inst, errlog, ldifname, bename=DEFAULT_BENAME, suffix=None): |
| 148 |
self.inst = inst |
| 149 |
self.errlog = errlog |
| 150 |
self.ldifname = ldifname |
| 151 |
self.bename = bename |
| 152 |
self.suffix = suffix |
| 153 |
self.ldif = ldifname if ldifname.startswith('/') else f'{inst.get_ldif_dir()}/{ldifname}.ldif' |
| 154 |
|
| 155 |
@abstractmethod |
| 156 |
def get_name(self): |
| 157 |
pass |
| 158 |
|
| 159 |
@abstractmethod |
| 160 |
def _run_task_b(self): |
| 161 |
pass |
| 162 |
|
| 163 |
@abstractmethod |
| 164 |
def _run_task_s(self): |
| 165 |
pass |
| 166 |
|
| 167 |
@abstractmethod |
| 168 |
def _run_offline(self): |
| 169 |
pass |
| 170 |
|
| 171 |
@abstractmethod |
| 172 |
def _set_log_pattern(self, success): |
| 173 |
pass |
| 174 |
|
| 175 |
def run(self, extra_checks, success=True): |
| 176 |
if self.errlog: |
| 177 |
self._set_log_pattern(success) |
| 178 |
self.errlog.seek2end() |
| 179 |
|
| 180 |
if self.inst.status(): |
| 181 |
if self.bename: |
| 182 |
log.info(f"Performing online {self.get_name()} of backend {self.bename} into LDIF file {self.ldif}") |
| 183 |
r = self._run_task_b() |
| 184 |
else: |
| 185 |
log.info(f"Performing online {self.get_name()} of suffix {self.suffix} into LDIF file {self.ldif}") |
| 186 |
r = self._run_task_s() |
| 187 |
r.wait() |
| 188 |
time.sleep(1) |
| 189 |
else: |
| 190 |
if self.bename: |
| 191 |
log.info(f"Performing offline {self.get_name()} of backend {self.bename} into LDIF file {self.ldif}") |
| 192 |
else: |
| 193 |
log.info(f"Performing offline {self.get_name()} of suffix {self.suffix} into LDIF file {self.ldif}") |
| 194 |
self._run_offline() |
| 195 |
if self.errlog: |
| 196 |
expected_counts = ['*' for _ in range(len(self.errlog.patterns))] |
| 197 |
for (idx, val) in extra_checks: |
| 198 |
expected_counts[idx] = val |
| 199 |
res = self.errlog.countCaptures() |
| 200 |
log.info(f'Expected errorlog counts are: {expected_counts}') |
| 201 |
if success is True or success is False: |
| 202 |
log.info(f'Number of {self.errlog.patterns[0]} in errorlog is: {res[0]}') |
| 203 |
assert res[0] >= 1 |
| 204 |
for (idx, val) in extra_checks: |
| 205 |
self.errlog.check(idx, val) |
| 206 |
|
| 207 |
def check_db(self): |
| 208 |
assert self.inst.dbscan(bename=self.bename, index='id2entry') |
| 209 |
|
| 210 |
|
| 211 |
class Importer(IEHandler): |
| 212 |
def get_name(self): |
| 213 |
return "import" |
| 214 |
|
| 215 |
def _set_log_pattern(self, success): |
| 216 |
if success is True: |
| 217 |
self.errlog.patterns[0] = 'import complete' |
| 218 |
elif success is False: |
| 219 |
self.errlog.patterns[0] = 'import failed' |
| 220 |
|
| 221 |
def _run_task_b(self): |
| 222 |
bes = Backends(self.inst) |
| 223 |
r = bes.import_ldif(self.bename, [self.ldif,], include_suffixes=self.suffix) |
| 224 |
return r |
| 225 |
|
| 226 |
def _run_task_s(self): |
| 227 |
r = ImportTask(self.inst) |
| 228 |
r.import_suffix_from_ldif(self.ldif, self.suffix) |
| 229 |
return r |
| 230 |
|
| 231 |
def _run_offline(self): |
| 232 |
log.info(f'self.inst.ldif2db({self.bename}, {self.suffix}, ...)') |
| 233 |
if self.suffix is None: |
| 234 |
self.inst.ldif2db(self.bename, self.suffix, None, False, self.ldif) |
| 235 |
else: |
| 236 |
self.inst.ldif2db(self.bename, [self.suffix, ], None, False, self.ldif) |
| 237 |
|
| 238 |
|
| 239 |
class Exporter(IEHandler): |
| 240 |
def get_name(self): |
| 241 |
return "export" |
| 242 |
|
| 243 |
def _set_log_pattern(self, success): |
| 244 |
if success is True: |
| 245 |
self.errlog.patterns[0] = 'export finished' |
| 246 |
elif success is False: |
| 247 |
self.errlog.patterns[0] = 'export failed' |
| 248 |
|
| 249 |
def _run_task_b(self): |
| 250 |
bes = Backends(self.inst) |
| 251 |
r = bes.export_ldif(self.bename, self.ldif, include_suffixes=self.suffix) |
| 252 |
return r |
| 253 |
|
| 254 |
def _run_task_s(self): |
| 255 |
r = ExportTask(self.inst) |
| 256 |
r.export_suffix_to_ldif(self.ldif, self.suffix) |
| 257 |
return r |
| 258 |
|
| 259 |
def _run_offline(self): |
| 260 |
self.inst.db2ldif(self.bename, self.suffix, None, False, False, self.ldif) |
| 261 |
|
| 262 |
|
| 263 |
def preserve_func(topo, request, restart): |
| 264 |
# Ensure that topology get preserved helper |
| 265 |
inst = topo.standalone |
| 266 |
|
| 267 |
def fin(): |
| 268 |
if restart: |
| 269 |
inst.restart() |
| 270 |
Importer(inst, None, "save").run(()) |
| 271 |
|
| 272 |
r = Exporter(inst, None, "save") |
| 273 |
if not os.path.isfile(r.ldif): |
| 274 |
r.run(()) |
| 275 |
request.addfinalizer(fin) |
| 276 |
|
| 277 |
|
| 278 |
@pytest.fixture(scope="function") |
| 279 |
def preserve(topo, request): |
| 280 |
# Ensure that topology get preserved (no restart) |
| 281 |
preserve_func(topo, request, False) |
| 282 |
|
| 283 |
|
| 284 |
@pytest.fixture(scope="function") |
| 285 |
def preserve_r(topo, request): |
| 286 |
# Ensure that topology get preserved (with restart) |
| 287 |
preserve_func(topo, request, True) |
| 288 |
|
| 289 |
|
| 290 |
@pytest.fixture(scope="function") |
| 291 |
def verify(topo): |
| 292 |
# Check that backend is not broken |
| 293 |
inst = topo.standalone |
| 294 |
dn=f'uid=demo_user,ou=people,{DEFAULT_SUFFIX}' |
| 295 |
assert UserAccount(inst,dn).exists() |
| 296 |
|
| 297 |
|
| 298 |
def test_replay_import_operation(topo, preserve_r, verify): |
| 299 |
""" Check after certain failed import operation, is it |
| 300 |
possible to replay an import operation |
| 301 |
|
| 302 |
:id: 5f5ca532-8e18-4f7b-86bc-ac585215a473 |
| 303 |
:feature: Import |
| 304 |
:setup: Standalone instance |
| 305 |
:steps: |
| 306 |
1. Export the backend into an ldif file |
| 307 |
2. Perform high load of operation on the server (Add/Del users) |
| 308 |
3. Perform an import operation |
| 309 |
4. Again perform an import operation (same as 3) |
| 310 |
:expectedresults: |
| 311 |
1. It should be successful |
| 312 |
2. It should be successful |
| 313 |
3. It should be unsuccessful, should give OPERATIONS_ERROR |
| 314 |
4. It should be successful now |
| 315 |
""" |
| 316 |
log.info("Exporting LDIF online...") |
| 317 |
ldif_dir = topo.standalone.get_ldif_dir() |
| 318 |
export_ldif = ldif_dir + '/export.ldif' |
| 319 |
|
| 320 |
r = ExportTask(topo.standalone) |
| 321 |
r.export_suffix_to_ldif(ldiffile=export_ldif, suffix=DEFAULT_SUFFIX) |
| 322 |
r.wait() |
| 323 |
add_del_users1 = AddDelUsers(topo) |
| 324 |
add_del_users1.start() |
| 325 |
|
| 326 |
log.info("Importing LDIF online, should raise operation error.") |
| 327 |
|
| 328 |
trials = 0 |
| 329 |
while not add_del_users1.has_started() and trials < 10: |
| 330 |
trials += 1 |
| 331 |
time.sleep(1) |
| 332 |
r = ImportTask(topo.standalone) |
| 333 |
try: |
| 334 |
r.import_suffix_from_ldif(ldiffile=export_ldif, suffix=DEFAULT_SUFFIX) |
| 335 |
except ldap.OPERATIONS_ERROR: |
| 336 |
break |
| 337 |
log.info(f'Looping. Tried {trials} times so far.') |
| 338 |
add_del_users1.stop() |
| 339 |
add_del_users1.join() |
| 340 |
|
| 341 |
log.info("Importing LDIF online") |
| 342 |
|
| 343 |
r = ImportTask(topo.standalone) |
| 344 |
r.import_suffix_from_ldif(ldiffile=export_ldif, suffix=DEFAULT_SUFFIX) |
| 345 |
|
| 346 |
|
| 347 |
def test_import_be_default(topo, verify): |
| 348 |
""" Create a backend using the name "default". previously this name was |
| 349 |
used int |
| 350 |
|
| 351 |
:id: 8e507beb-e917-4330-8cac-1ff0eee10508 |
| 352 |
:feature: Import |
| 353 |
:setup: Standalone instance |
| 354 |
:steps: |
| 355 |
1. Create a test suffix using the be name of "default" |
| 356 |
2. Create an ldif for the "default" backend |
| 357 |
3. Import ldif |
| 358 |
4. Verify all entries were imported |
| 359 |
:expectedresults: |
| 360 |
1. Success |
| 361 |
2. Success |
| 362 |
3. Success |
| 363 |
4. Success |
| 364 |
""" |
| 365 |
log.info('Adding suffix:{} and backend: {}...'.format(TEST_DEFAULT_SUFFIX, |
| 366 |
TEST_DEFAULT_NAME)) |
| 367 |
backends = Backends(topo.standalone) |
| 368 |
backends.create(properties={'nsslapd-suffix': TEST_DEFAULT_SUFFIX, |
| 369 |
'name': TEST_DEFAULT_NAME}) |
| 370 |
|
| 371 |
log.info('Create LDIF file and import it...') |
| 372 |
ldif_dir = topo.standalone.get_ldif_dir() |
| 373 |
ldif_file = os.path.join(ldif_dir, 'default.ldif') |
| 374 |
dbgen_users(topo.standalone, 5, ldif_file, TEST_DEFAULT_SUFFIX) |
| 375 |
|
| 376 |
log.info('Stopping the server and running offline import...') |
| 377 |
topo.standalone.stop() |
| 378 |
assert topo.standalone.ldif2db(TEST_DEFAULT_NAME, None, None, |
| 379 |
None, ldif_file) |
| 380 |
topo.standalone.start() |
| 381 |
|
| 382 |
log.info('Verifying entry count after import...') |
| 383 |
entries = topo.standalone.search_s(TEST_DEFAULT_SUFFIX, |
| 384 |
ldap.SCOPE_SUBTREE, |
| 385 |
"(objectclass=*)") |
| 386 |
assert len(entries) > 1 |
| 387 |
|
| 388 |
log.info('Test PASSED') |
| 389 |
|
| 390 |
|
| 391 |
def test_del_suffix_import(topo, verify): |
| 392 |
"""Adding a database entry fails if the same database was deleted after an import |
| 393 |
|
| 394 |
:id: 652421ef-738b-47ed-80ec-2ceece6b5d77 |
| 395 |
:feature: Import |
| 396 |
:setup: Standalone instance |
| 397 |
:steps: 1. Create a test suffix and add few entries |
| 398 |
2. Stop the server and do offline import using ldif2db |
| 399 |
3. Delete the suffix backend |
| 400 |
4. Add a new suffix with the same database name |
| 401 |
5. Check if adding the same database name is a success |
| 402 |
:expectedresults: Adding database with the same name should be successful |
| 403 |
""" |
| 404 |
|
| 405 |
log.info('Adding suffix:{} and backend: {}'.format(TEST_SUFFIX1, TEST_BACKEND1)) |
| 406 |
backends = Backends(topo.standalone) |
| 407 |
backend = backends.create(properties={'nsslapd-suffix': TEST_SUFFIX1, |
| 408 |
'name': TEST_BACKEND1}) |
| 409 |
|
| 410 |
log.info('Create LDIF file and import it') |
| 411 |
ldif_dir = topo.standalone.get_ldif_dir() |
| 412 |
ldif_file = os.path.join(ldif_dir, 'suffix_del1.ldif') |
| 413 |
|
| 414 |
dbgen_users(topo.standalone, 10, ldif_file, TEST_SUFFIX1) |
| 415 |
|
| 416 |
log.info('Stopping the server and running offline import') |
| 417 |
topo.standalone.stop() |
| 418 |
assert topo.standalone.ldif2db(TEST_BACKEND1, TEST_SUFFIX1, None, None, ldif_file) |
| 419 |
topo.standalone.start() |
| 420 |
|
| 421 |
log.info('Deleting suffix-{}'.format(TEST_SUFFIX2)) |
| 422 |
backend.delete() |
| 423 |
|
| 424 |
log.info('Adding the same database-{} after deleting it'.format(TEST_BACKEND1)) |
| 425 |
backends.create(properties={'nsslapd-suffix': TEST_SUFFIX1, |
| 426 |
'name': TEST_BACKEND1}) |
| 427 |
|
| 428 |
|
| 429 |
def test_del_suffix_backend(topo, verify): |
| 430 |
"""Adding a database entry fails if the same database was deleted after an import |
| 431 |
|
| 432 |
:id: ac702c35-74b6-434e-8e30-316433f3e91a |
| 433 |
:feature: Import |
| 434 |
:setup: Standalone instance |
| 435 |
:steps: 1. Create a test suffix and add entries |
| 436 |
2. Stop the server and do online import using ldif2db |
| 437 |
3. Delete the suffix backend |
| 438 |
4. Add a new suffix with the same database name |
| 439 |
5. Restart the server and check the status |
| 440 |
:expectedresults: Adding database with the same name should be successful and the server should not hang |
| 441 |
""" |
| 442 |
|
| 443 |
log.info('Adding suffix:{} and backend: {}'.format(TEST_SUFFIX2, TEST_BACKEND2)) |
| 444 |
backends = Backends(topo.standalone) |
| 445 |
backend = backends.create(properties={'nsslapd-suffix': TEST_SUFFIX2, |
| 446 |
'name': TEST_BACKEND2}) |
| 447 |
|
| 448 |
log.info('Create LDIF file and import it') |
| 449 |
ldif_dir = topo.standalone.get_ldif_dir() |
| 450 |
ldif_file = os.path.join(ldif_dir, 'suffix_del2.ldif') |
| 451 |
|
| 452 |
dbgen_users(topo.standalone, 10, ldif_file, TEST_SUFFIX2) |
| 453 |
|
| 454 |
topo.standalone.tasks.importLDIF(suffix=TEST_SUFFIX2, input_file=ldif_file, args={TASK_WAIT: True}) |
| 455 |
|
| 456 |
log.info('Deleting suffix-{}'.format(TEST_SUFFIX2)) |
| 457 |
backend.delete() |
| 458 |
|
| 459 |
log.info('Adding the same database-{} after deleting it'.format(TEST_BACKEND2)) |
| 460 |
backends.create(properties={'nsslapd-suffix': TEST_SUFFIX2, |
| 461 |
'name': TEST_BACKEND2}) |
| 462 |
log.info('Checking if server can be restarted after re-adding the same database') |
| 463 |
topo.standalone.restart() |
| 464 |
assert not topo.standalone.detectDisorderlyShutdown() |
| 465 |
|
| 466 |
|
| 467 |
def test_import_duplicate_dn(topo, preserve, verify): |
| 468 |
"""Import ldif with duplicate DNs, should not log error "unable to flush" |
| 469 |
|
| 470 |
:id: dce2b898-119d-42b8-a236-1130f58bff17 |
| 471 |
:setup: Standalone instance, ldif file with duplicate entries |
| 472 |
:steps: |
| 473 |
1. Create a ldif file with duplicate entries |
| 474 |
2. Import ldif file to DS |
| 475 |
3. Check error log file, it should not log "unable to flush" |
| 476 |
4. Check error log file, it should log "Duplicated DN detected" |
| 477 |
:expectedresults: |
| 478 |
1. Success |
| 479 |
2. Success |
| 480 |
3. Success |
| 481 |
4. Success |
| 482 |
""" |
| 483 |
|
| 484 |
standalone = topo.standalone |
| 485 |
|
| 486 |
log.info('Delete the previous error logs') |
| 487 |
standalone.deleteErrorLogs() |
| 488 |
|
| 489 |
log.info('Create import file') |
| 490 |
l = """dn: dc=example,dc=com |
| 491 |
objectclass: top |
| 492 |
objectclass: domain |
| 493 |
dc: example |
| 494 |
|
| 495 |
dn: ou=myDups00001,dc=example,dc=com |
| 496 |
objectclass: top |
| 497 |
objectclass: organizationalUnit |
| 498 |
ou: myDups00001 |
| 499 |
|
| 500 |
dn: ou=myDups00001,dc=example,dc=com |
| 501 |
objectclass: top |
| 502 |
objectclass: organizationalUnit |
| 503 |
ou: myDups00001 |
| 504 |
""" |
| 505 |
|
| 506 |
ldif_dir = standalone.get_ldif_dir() |
| 507 |
ldif_file = os.path.join(ldif_dir, 'data.ldif') |
| 508 |
with open(ldif_file, "w") as fd: |
| 509 |
fd.write(l) |
| 510 |
fd.close() |
| 511 |
os.chmod(ldif_file, 0o777) |
| 512 |
|
| 513 |
log.info('Import ldif with duplicate entry') |
| 514 |
assert standalone.tasks.importLDIF(suffix=DEFAULT_SUFFIX, input_file=ldif_file, args={TASK_WAIT: True}) |
| 515 |
|
| 516 |
log.info('Restart the server to flush the logs') |
| 517 |
standalone.restart() |
| 518 |
|
| 519 |
log.info('Error log should not have "unable to flush" message') |
| 520 |
assert not standalone.ds_error_log.match('.*unable to flush.*') |
| 521 |
|
| 522 |
log.info('Error log should have "Duplicated DN detected" message') |
| 523 |
assert standalone.ds_error_log.match('.*Duplicated DN detected.*') |
| 524 |
|
| 525 |
@pytest.mark.tier2 |
| 526 |
@pytest.mark.skipif(not _check_disk_space(), reason="not enough disk space for lmdb map") |
| 527 |
@pytest.mark.xfail(ds_is_older("1.3.10.1"), reason="bz1749595 not fixed on versions older than 1.3.10.1") |
| 528 |
def test_large_ldif2db_ancestorid_index_creation(topo, _set_mdb_map_size, verify): |
| 529 |
"""Import with ldif2db a large file - check that the ancestorid index creation phase has a correct performance |
| 530 |
|
| 531 |
:id: fe7f78f6-6e60-425d-ad47-b39b67e29113 |
| 532 |
:setup: Standalone instance |
| 533 |
:steps: |
| 534 |
1. Delete the previous errors log to start from a fresh one |
| 535 |
2. Create test suffix and backend |
| 536 |
3. Create a large nested ldif file |
| 537 |
4. Stop the server |
| 538 |
5. Run an offline import |
| 539 |
6. Restart the server |
| 540 |
7. Check in the errors log that an independant ancestorid IDs sorting is done |
| 541 |
8. Get the log of the starting of the ancestorid indexing process |
| 542 |
9. Get the log of the end of the ancestorid indexing process |
| 543 |
10. Get the start and end time for ancestorid index creation from these logs |
| 544 |
11. Calculate the duration of the ancestorid indexing process |
| 545 |
:expectedresults: |
| 546 |
1. Success |
| 547 |
2. Test suffix and backend successfully created |
| 548 |
3. ldif file successfully created |
| 549 |
4. Success |
| 550 |
5. Import is successfully performed |
| 551 |
6. Success |
| 552 |
7. Log of ancestorid sorting start and end are present |
| 553 |
8. Log of the beginning of gathering ancestorid is found |
| 554 |
9. Log of the final ancestorid index creation is found |
| 555 |
10. Start and end times are successfully extracted |
| 556 |
11. The duration of the ancestorid index creation process should be less than 10s |
| 557 |
""" |
| 558 |
|
| 559 |
ldif_dir = topo.standalone.get_ldif_dir() |
| 560 |
ldif_file = os.path.join(topo.standalone.ds_paths.ldif_dir, 'large_nested.ldif') |
| 561 |
|
| 562 |
# Have a reasonable balance between the need for a large ldif file to import and the time of test execution |
| 563 |
# total number of users |
| 564 |
num_users = 100000 |
| 565 |
|
| 566 |
# Choose a limited number of users per node to get as much as possible non-leaf entries |
| 567 |
node_limit = 5 |
| 568 |
|
| 569 |
# top suffix |
| 570 |
suffix = 'o=test' |
| 571 |
|
| 572 |
# backend |
| 573 |
backend = 'test' |
| 574 |
|
| 575 |
log.info('Delete the previous errors logs') |
| 576 |
topo.standalone.deleteErrorLogs() |
| 577 |
|
| 578 |
log.info('Add suffix:{} and backend: {}...'.format(suffix, backend)) |
| 579 |
|
| 580 |
backends = Backends(topo.standalone) |
| 581 |
backends.create(properties={'nsslapd-suffix': suffix, |
| 582 |
'name': backend}) |
| 583 |
|
| 584 |
props = { |
| 585 |
'numUsers' : num_users, |
| 586 |
'nodeLimit' : node_limit, |
| 587 |
'suffix' : suffix |
| 588 |
} |
| 589 |
instance = topo.standalone |
| 590 |
|
| 591 |
log.info('Create a large nested ldif file using dbgen : %s' % ldif_file) |
| 592 |
dbgen_nested_ldif(instance, ldif_file, props) |
| 593 |
|
| 594 |
log.info('Stop the server and run offline import...') |
| 595 |
topo.standalone.stop() |
| 596 |
assert topo.standalone.ldif2db(backend, None, None, |
| 597 |
None, ldif_file) |
| 598 |
|
| 599 |
log.info('Starting the server') |
| 600 |
topo.standalone.start() |
| 601 |
|
| 602 |
# With lmdb there is no more any special phase for ancestorid |
| 603 |
# because ancestorsid get updated on the fly while processing the |
| 604 |
# entryrdn (by up the parents chain to compute the parentid |
| 605 |
# |
| 606 |
# But there is still a numSubordinates generation phase |
| 607 |
if get_default_db_lib() == "mdb": |
| 608 |
log.info('parse the errors logs to check lines with "Generating numSubordinates complete." are present') |
| 609 |
end_numsubordinates = str(topo.standalone.ds_error_log.match(r'.*Generating numSubordinates complete.*'))[1:-1] |
| 610 |
assert len(end_numsubordinates) > 0 |
| 611 |
|
| 612 |
else: |
| 613 |
log.info('parse the errors logs to check lines with "Starting sort of ancestorid" are present') |
| 614 |
start_sort_str = str(topo.standalone.ds_error_log.match(r'.*Starting sort of ancestorid non-leaf IDs*'))[1:-1] |
| 615 |
assert len(start_sort_str) > 0 |
| 616 |
|
| 617 |
log.info('parse the errors logs to check lines with "Finished sort of ancestorid" are present') |
| 618 |
end_sort_str = str(topo.standalone.ds_error_log.match(r'.*Finished sort of ancestorid non-leaf IDs*'))[1:-1] |
| 619 |
assert len(end_sort_str) > 0 |
| 620 |
|
| 621 |
log.info('parse the error logs for the line with "Gathering ancestorid non-leaf IDs"') |
| 622 |
start_ancestorid_indexing_op_str = str(topo.standalone.ds_error_log.match(r'.*Gathering ancestorid non-leaf IDs*'))[1:-1] |
| 623 |
assert len(start_ancestorid_indexing_op_str) > 0 |
| 624 |
|
| 625 |
log.info('parse the error logs for the line with "Created ancestorid index"') |
| 626 |
end_ancestorid_indexing_op_str = str(topo.standalone.ds_error_log.match(r'.*Created ancestorid index*'))[1:-1] |
| 627 |
assert len(end_ancestorid_indexing_op_str) > 0 |
| 628 |
|
| 629 |
log.info('get the ancestorid non-leaf IDs indexing start and end time from the collected strings') |
| 630 |
# Collected lines look like : '[15/May/2020:05:30:27.245967313 -0400] - INFO - bdb_get_nonleaf_ids - import userRoot: Gathering ancestorid non-leaf IDs...' |
| 631 |
# We are getting the sec.nanosec part of the date, '27.245967313' in the above example |
| 632 |
start_time = (start_ancestorid_indexing_op_str.split()[0]).split(':')[3] |
| 633 |
end_time = (end_ancestorid_indexing_op_str.split()[0]).split(':')[3] |
| 634 |
|
| 635 |
log.info('Calculate the elapsed time for the ancestorid non-leaf IDs index creation') |
| 636 |
etime = (Decimal(end_time) - Decimal(start_time)) |
| 637 |
# The time for the ancestorid index creation should be less than 10s for an offline import of an ldif file with 100000 entries / 5 entries per node |
| 638 |
# Should be adjusted if these numbers are modified in the test |
| 639 |
assert etime <= 10 |
| 640 |
|
| 641 |
|
| 642 |
def create_backend_and_import(instance, ldif_file, suffix, backend): |
| 643 |
log.info(f'Add suffix:{suffix} and backend: {backend}...') |
| 644 |
backends = Backends(instance) |
| 645 |
backends.create(properties={'nsslapd-suffix': suffix, 'name': backend}) |
| 646 |
props = {'numUsers': 10000, 'nodeLimit': 5, 'suffix': suffix} |
| 647 |
|
| 648 |
log.info(f'Create a large nested ldif file using dbgen : {ldif_file}') |
| 649 |
dbgen_nested_ldif(instance, ldif_file, props) |
| 650 |
|
| 651 |
log.info('Stop the server and run offline import...') |
| 652 |
instance.stop() |
| 653 |
log.info('Measure the import time for the ldif file...') |
| 654 |
start = time.time() |
| 655 |
assert instance.ldif2db(backend, None, None, None, ldif_file) |
| 656 |
end = time.time() |
| 657 |
instance.start() |
| 658 |
return end - start |
| 659 |
|
| 660 |
|
| 661 |
@pytest.mark.skipif(get_default_db_lib() == "mdb", reason="Not cache size over mdb") |
| 662 |
def test_ldif2db_after_backend_create(topo, verify): |
| 663 |
"""Test that ldif2db after backend creation is not slow first time |
| 664 |
|
| 665 |
:id: c1ab1df7-c70a-46be-bbca-8d65c6ebaa14 |
| 666 |
:setup: Standalone Instance |
| 667 |
:steps: |
| 668 |
1. Create backend and suffix |
| 669 |
2. Generate large LDIF file |
| 670 |
3. Stop server and run offline import |
| 671 |
4. Measure import time |
| 672 |
5. Restart server and repeat steps 1-4 with new backend and suffix |
| 673 |
:expectedresults: |
| 674 |
1. Operation successful |
| 675 |
2. Operation successful |
| 676 |
3. Operation successful |
| 677 |
4. Import times should be approximately the same |
| 678 |
5. Operation successful |
| 679 |
""" |
| 680 |
|
| 681 |
instance = topo.standalone |
| 682 |
ldif_dir = instance.get_ldif_dir() |
| 683 |
ldif_file_1 = os.path.join(ldif_dir, 'large_nested_1.ldif') |
| 684 |
ldif_file_2 = os.path.join(ldif_dir, 'large_nested_2.ldif') |
| 685 |
|
| 686 |
import_time_1 = create_backend_and_import(instance, ldif_file_1, 'o=test_1', 'test_1') |
| 687 |
import_time_2 = create_backend_and_import(instance, ldif_file_2, 'o=test_2', 'test_2') |
| 688 |
|
| 689 |
log.info('Import times should be approximately the same') |
| 690 |
assert abs(import_time_1 - import_time_2) < 20 |
| 691 |
|
| 692 |
|
| 693 |
def test_ldif_missing_suffix_entry(topo, request, verify): |
| 694 |
"""Test that ldif2db/import aborts if suffix entry is not in the ldif |
| 695 |
|
| 696 |
:id: 731bd0d6-8cc8-11f0-8ef2-c85309d5c3e3 |
| 697 |
:setup: Standalone Instance |
| 698 |
:steps: |
| 699 |
1. Prepare final cleanup |
| 700 |
2. Add a few users |
| 701 |
3. Export ou=people subtree |
| 702 |
4. Online import using backend name ou=people subtree |
| 703 |
5. Online import using suffix name ou=people subtree |
| 704 |
6. Stop the instance |
| 705 |
7. Offline import using backend name ou=people subtree |
| 706 |
8. Offline import using suffix name ou=people subtree |
| 707 |
9. Generate ldif with a far away suffix |
| 708 |
10. Offline import using backend name and "far" ldif |
| 709 |
11. Offline import using suffix name and "far" ldif |
| 710 |
12. Start the instance |
| 711 |
13. Online import using backend name and "far" ldif |
| 712 |
14. Online import using suffix name and "far" ldif |
| 713 |
:expectedresults: |
| 714 |
1. Operation successful |
| 715 |
2. Operation successful |
| 716 |
3. Operation successful |
| 717 |
4. Import should success, skip all entries, db should exists |
| 718 |
5. Import should success, skip all entries, db should exists |
| 719 |
6. Operation successful |
| 720 |
7. Import should success, skip all entries, db should exists |
| 721 |
8. Import should success, skip all entries, db should exists |
| 722 |
9. Operation successful |
| 723 |
10. Import should success, skip all entries, db should exists |
| 724 |
11. Import should success, 10 entries skipped, db should exists |
| 725 |
12. Operation successful |
| 726 |
13. Import should success, skip all entries, db should exists |
| 727 |
14. Import should success, 10 entries skipped, db should exists |
| 728 |
""" |
| 729 |
|
| 730 |
inst = topo.standalone |
| 731 |
inst.config.set('nsslapd-errorlog-level', '266354688') |
| 732 |
no_suffix_on = ( |
| 733 |
(1, 0), # no errors are expected. |
| 734 |
(2, 1), # 1 warning is expected. |
| 735 |
(3, 0), # no 'no parent' warning is expected. |
| 736 |
(4, 1), # 1 'all entries were skipped' warning |
| 737 |
(5, 0), # no 'returning task warning' info message |
| 738 |
) |
| 739 |
no_suffix_off = ( |
| 740 |
(1, 0), # no errors are expected. |
| 741 |
(2, 1), # 1 warning is expected. |
| 742 |
(3, 0), # no 'no parent' warning is expected. |
| 743 |
(4, 1), # 1 'all entries were skipped' warning |
| 744 |
(5, 1), # 1 'returning task warning' info message |
| 745 |
) |
| 746 |
|
| 747 |
far_suffix_on = ( |
| 748 |
(1, 0), # no errors are expected. |
| 749 |
(2, 1), # 1 warning (consolidated, pre-check aborts after 4 entries) |
| 750 |
(3, 0), # 0 'no parent' warnings (pre-check aborts before processing) |
| 751 |
(4, 1), # 1 'all entries were skipped' warning (from pre-check) |
| 752 |
(5, 0), # 0 'returning task warning' info message (online import) |
| 753 |
) |
| 754 |
# Backend-specific behavior for orphan detection when suffix parameter is provided |
| 755 |
nbw = 0 if get_default_db_lib() == "bdb" else 10 |
| 756 |
far_suffix_with_suffix_on = ( |
| 757 |
(1, 0), # no errors are expected. |
| 758 |
(2, nbw), # 0 (BDB early filtering) or 10 (LMDB orphan detection) warnings |
| 759 |
(3, nbw), # 0 (BDB early filtering) or 10 (LMDB orphan detection) 'no parent' warnings |
| 760 |
(4, 0), # 0 'all entries were skipped' warning (no pre-check abort) |
| 761 |
(5, 0), # 0 'returning task warning' info message (online import) |
| 762 |
) |
| 763 |
far_suffix_off = ( |
| 764 |
(1, 0), # no errors are expected. |
| 765 |
(2, 1), # 1 warning (consolidated, pre-check detects missing suffix) |
| 766 |
(3, 0), # 0 'no parent' warnings (pre-check aborts before processing) |
| 767 |
(4, 1), # 1 'all entries were skipped' warning (from pre-check) |
| 768 |
(5, 1), # 1 'returning task warning' info message (offline import) |
| 769 |
) |
| 770 |
far_suffix_with_suffix_off = ( |
| 771 |
(1, 0), # no errors are expected. |
| 772 |
(2, nbw), # 0 (BDB early filtering) or 10 (LMDB orphan detection) warnings |
| 773 |
(3, nbw), # 0 (BDB early filtering) or 10 (LMDB orphan detection) 'no parent' warnings |
| 774 |
(4, 0), # 0 'all entries were skipped' warning (no pre-check abort) |
| 775 |
(5, 0), # 0 'returning task warning' (rc=0, successful import of suffix) |
| 776 |
) |
| 777 |
|
| 778 |
with open(inst.ds_paths.error_log, 'at+') as fd: |
| 779 |
patterns = ( |
| 780 |
"Reserved for IEHandler", |
| 781 |
" ERR ", |
| 782 |
" WARN ", |
| 783 |
"has no parent", |
| 784 |
"all entries were skipped", |
| 785 |
"returning task warning", |
| 786 |
) |
| 787 |
errlog = LogHandler(fd, patterns) |
| 788 |
no_errors = ((1, 0), (2, 0)) # no errors nor warnings are expected. |
| 789 |
|
| 790 |
|
| 791 |
# 1. Prepare final cleanup |
| 792 |
Exporter(inst, errlog, "full").run(no_errors) |
| 793 |
|
| 794 |
def fin(): |
| 795 |
inst.start() |
| 796 |
with open(inst.ds_paths.error_log, 'at+') as cleanup_fd: |
| 797 |
cleanup_errlog = LogHandler(cleanup_fd, patterns) |
| 798 |
Importer(inst, cleanup_errlog, "full").run(no_errors) |
| 799 |
|
| 800 |
if not DEBUGGING: |
| 801 |
request.addfinalizer(fin) |
| 802 |
|
| 803 |
# 2. Add a few users |
| 804 |
user = UserAccounts(inst, DEFAULT_SUFFIX) |
| 805 |
users = [ user.create_test_user(uid=i) for i in range(10) ] |
| 806 |
|
| 807 |
# 3. Export ou=people subtree |
| 808 |
e = Exporter(inst, errlog, "people", suffix=f'ou=people,{DEFAULT_SUFFIX}') |
| 809 |
e.run(no_errors) # no errors nor warnings are expected. |
| 810 |
|
| 811 |
# 4. Online import using backend name ou=people subtree |
| 812 |
e = Importer(inst, errlog, "people") |
| 813 |
e.run(no_suffix_on) |
| 814 |
e.check_db() |
| 815 |
|
| 816 |
# 5. Online import using suffix name ou=people subtree |
| 817 |
e = Importer(inst, errlog, "people", suffix=DEFAULT_SUFFIX) |
| 818 |
e.run(no_suffix_on) |
| 819 |
e.check_db() |
| 820 |
|
| 821 |
# 6. Stop the instance |
| 822 |
inst.stop() |
| 823 |
|
| 824 |
# 7. Offline import using backend name ou=people subtree |
| 825 |
e = Importer(inst, errlog, "people") |
| 826 |
e.run(no_suffix_off) |
| 827 |
e.check_db() |
| 828 |
|
| 829 |
# 8. Offline import using suffix name ou=people subtree |
| 830 |
e = Importer(inst, errlog, "people", suffix=DEFAULT_SUFFIX) |
| 831 |
e.run(no_suffix_off) |
| 832 |
e.check_db() |
| 833 |
|
| 834 |
# 9. Generate ldif with a far away suffix |
| 835 |
e = Importer(inst, errlog, "full") |
| 836 |
people_ldif = e.ldif |
| 837 |
e = Importer(inst, errlog, "far") |
| 838 |
with open(e.ldif, "wt") as fout: |
| 839 |
with open(people_ldif, "rt") as fin: |
| 840 |
# Copy version |
| 841 |
line = fin.readline() |
| 842 |
fout.write(line) |
| 843 |
line = fin.readline() |
| 844 |
fout.write(line) |
| 845 |
# Generate fake entries |
| 846 |
for idx in range(10): |
| 847 |
fout.write(f"dn: uid=id{idx},dc=foo\nobjectclasses: extensibleObject\n\n") |
| 848 |
for line in iter(fin.readline, ''): |
| 849 |
fout.write(line) |
| 850 |
|
| 851 |
os.chmod(e.ldif, 0o644) |
| 852 |
|
| 853 |
# 10. Offline import using backend name ou=people subtree |
| 854 |
e.run(far_suffix_off) |
| 855 |
e.check_db() |
| 856 |
|
| 857 |
# 11. Offline import using suffix name ou=people subtree |
| 858 |
e = Importer(inst, errlog, "far", suffix=DEFAULT_SUFFIX) |
| 859 |
e.run(far_suffix_with_suffix_off) |
| 860 |
e.check_db() |
| 861 |
|
| 862 |
# 12. Start the instance |
| 863 |
inst.start() |
| 864 |
|
| 865 |
# 13. Online import using backend name ou=people subtree |
| 866 |
e = Importer(inst, errlog, "far") |
| 867 |
e.run(far_suffix_on) |
| 868 |
e.check_db() |
| 869 |
|
| 870 |
# 14. Online import using suffix name ou=people subtree |
| 871 |
e = Importer(inst, errlog, "far", suffix=DEFAULT_SUFFIX) |
| 872 |
e.run(far_suffix_with_suffix_on) |
| 873 |
e.check_db() |
| 874 |
|
| 875 |
|
| 876 |
if __name__ == '__main__': |
| 877 |
# Run isolated |
| 878 |
# -s for DEBUG mode |
| 879 |
CURRENT_FILE = os.path.realpath(__file__) |
| 880 |
pytest.main("-s {}".format(CURRENT_FILE)) |