| 1 |
# --- BEGIN COPYRIGHT BLOCK --- |
| 2 |
# Copyright (C) 2022 Red Hat, Inc. |
| 3 |
# All rights reserved. |
| 4 |
# |
| 5 |
# License: GPL (version 3 or any later version). |
| 6 |
# See LICENSE for details. |
| 7 |
# --- END COPYRIGHT BLOCK --- |
| 8 |
# |
| 9 |
import logging |
| 10 |
import os |
| 11 |
import shutil |
| 12 |
import time |
| 13 |
import glob |
| 14 |
import subprocess |
| 15 |
import pytest |
| 16 |
import ldap |
| 17 |
from datetime import datetime |
| 18 |
from lib389._constants import DN_DM, PASSWORD, DEFAULT_SUFFIX, INSTALL_LATEST_CONFIG |
| 19 |
from lib389.properties import BACKEND_SAMPLE_ENTRIES, TASK_WAIT |
| 20 |
from test389.topologies import topology_st as topo, topology_m2 as topo_m2, set_timeout |
| 21 |
from lib389.backend import Backends, Backend |
| 22 |
from lib389.dbgen import dbgen_users |
| 23 |
from lib389.tasks import BackupTask, RestoreTask |
| 24 |
from lib389.config import BDB_LDBMConfig |
| 25 |
from lib389.idm.nscontainer import nsContainers |
| 26 |
from lib389 import DSEldif |
| 27 |
from lib389.utils import ds_is_older, get_default_db_lib |
| 28 |
from lib389.replica import ReplicationManager |
| 29 |
from threading import Thread, Event |
| 30 |
import tempfile |
| 31 |
|
| 32 |
|
| 33 |
|
| 34 |
pytestmark = pytest.mark.tier1 |
| 35 |
|
| 36 |
DEBUGGING = os.getenv("DEBUGGING", default=False) |
| 37 |
if DEBUGGING: |
| 38 |
logging.getLogger(__name__).setLevel(logging.DEBUG) |
| 39 |
else: |
| 40 |
logging.getLogger(__name__).setLevel(logging.INFO) |
| 41 |
log = logging.getLogger(__name__) |
| 42 |
|
| 43 |
# test_online_backup_and_dse_write may hang if 6372 is not fixed |
| 44 |
# so lets use a shorter timeout |
| 45 |
set_timeout(30*60) |
| 46 |
|
| 47 |
event = Event() |
| 48 |
|
| 49 |
BESTRUCT = [ |
| 50 |
{ "bename" : "be1", "suffix": "dc=be1", "nbusers": 1000 }, |
| 51 |
{ "bename" : "be2", "suffix": "dc=be2", "nbusers": 1000 }, |
| 52 |
{ "bename" : "be3", "suffix": "dc=be3", "nbusers": 1000 }, |
| 53 |
] |
| 54 |
|
| 55 |
class DseConfigContextManager: |
| 56 |
"""Change a config parameter is dse.ldif and restore it afterwards.""" |
| 57 |
|
| 58 |
def __init__(self, inst, dn, attr, value): |
| 59 |
self.inst = inst |
| 60 |
self.dn = dn |
| 61 |
self.attr = attr |
| 62 |
self.value = value |
| 63 |
self.oldvalue = None |
| 64 |
self.dseldif = DSEldif(inst) |
| 65 |
|
| 66 |
def __enter__(self): |
| 67 |
self.inst.stop() |
| 68 |
self.oldvalue = self.dseldif.get(self.dn, self.attr, single=True) |
| 69 |
self.dseldif.replace(self.dn, self.attr, self.value) |
| 70 |
log.info(f"Switching {self.dn}:{self.attr} to {self.value}") |
| 71 |
self.inst.start() |
| 72 |
|
| 73 |
def __exit__(self, exc_type, exc_value, exc_tb): |
| 74 |
self.inst.stop() |
| 75 |
log.info(f"Switching {self.dn}:{self.attr} to {self.oldvalue}") |
| 76 |
self.dseldif.replace(self.dn, self.attr, self.oldvalue) |
| 77 |
self.inst.start() |
| 78 |
|
| 79 |
|
| 80 |
@pytest.fixture(scope="function") |
| 81 |
def mytopo(topo, request): |
| 82 |
bes = [] |
| 83 |
|
| 84 |
def fin(): |
| 85 |
for be in bes: |
| 86 |
be.delete() |
| 87 |
for bak_dir in glob.glob(f'{inst.ds_paths.backup_dir}/*'): |
| 88 |
shutil.rmtree(bak_dir) |
| 89 |
|
| 90 |
if not DEBUGGING: |
| 91 |
request.addfinalizer(fin) |
| 92 |
|
| 93 |
inst = topo.standalone |
| 94 |
|
| 95 |
ldif_files = {} |
| 96 |
for d in BESTRUCT: |
| 97 |
bename = d['bename'] |
| 98 |
suffix = d['suffix'] |
| 99 |
nbusers = d['nbusers'] |
| 100 |
log.info(f'Adding suffix: {suffix} and backend: {bename}...') |
| 101 |
backends = Backends(inst) |
| 102 |
try: |
| 103 |
be = backends.create(properties={'nsslapd-suffix': suffix, 'name': bename}) |
| 104 |
# Insert at list head so that children backends get deleted before parent one. |
| 105 |
bes.insert(0, be) |
| 106 |
except ldap.UNWILLING_TO_PERFORM as e: |
| 107 |
if str(e) == "Mapping tree for this suffix exists!": |
| 108 |
pass |
| 109 |
else: |
| 110 |
raise e |
| 111 |
|
| 112 |
ldif_dir = inst.get_ldif_dir() |
| 113 |
ldif_files[bename] = os.path.join(ldif_dir, f'default_{bename}.ldif') |
| 114 |
dbgen_users(inst, nbusers, ldif_files[bename], suffix) |
| 115 |
inst.stop() |
| 116 |
for d in BESTRUCT: |
| 117 |
bename = d['bename'] |
| 118 |
inst.ldif2db(bename, None, None, None, ldif_files[bename]) |
| 119 |
inst.start() |
| 120 |
return topo |
| 121 |
|
| 122 |
|
| 123 |
def test_missing_backend(topo): |
| 124 |
"""Test that an error is returned when a restore is performed for a |
| 125 |
backend that is no longer present. |
| 126 |
|
| 127 |
:id: 889b8028-35cf-41d7-91f6-bc5193683646 |
| 128 |
:setup: Standalone Instance |
| 129 |
:steps: |
| 130 |
1. Create a second backend |
| 131 |
2. Perform a back up |
| 132 |
3. Remove one of the backends from the config |
| 133 |
4. Perform a restore |
| 134 |
:expectedresults: |
| 135 |
1. Success |
| 136 |
2. Success |
| 137 |
3. Success |
| 138 |
4. Failure |
| 139 |
""" |
| 140 |
|
| 141 |
# Create a new backend |
| 142 |
BE_NAME = 'backupRoot' |
| 143 |
BE_SUFFIX = 'dc=back,dc=up' |
| 144 |
props = { |
| 145 |
'cn': BE_NAME, |
| 146 |
'nsslapd-suffix': BE_SUFFIX, |
| 147 |
BACKEND_SAMPLE_ENTRIES: INSTALL_LATEST_CONFIG |
| 148 |
} |
| 149 |
be = Backend(topo.standalone) |
| 150 |
backend_entry = be.create(properties=props) |
| 151 |
|
| 152 |
# perform backup |
| 153 |
backup_dir_name = "backup-%s" % datetime.now().strftime("%Y_%m_%d_%H_%M_%S") |
| 154 |
archive = os.path.join(topo.standalone.ds_paths.backup_dir, backup_dir_name) |
| 155 |
backup_task = BackupTask(topo.standalone) |
| 156 |
task_properties = {'nsArchiveDir': archive} |
| 157 |
backup_task.create(properties=task_properties) |
| 158 |
backup_task.wait() |
| 159 |
assert backup_task.get_exit_code() == 0 |
| 160 |
|
| 161 |
# Remove new backend |
| 162 |
backend_entry.delete() |
| 163 |
|
| 164 |
# Restore the backup - it should fail |
| 165 |
restore_task = RestoreTask(topo.standalone) |
| 166 |
task_properties = {'nsArchiveDir': archive} |
| 167 |
restore_task.create(properties=task_properties) |
| 168 |
restore_task.wait() |
| 169 |
assert restore_task.get_exit_code() != 0 |
| 170 |
|
| 171 |
|
| 172 |
@pytest.mark.skipif(ds_is_older('1.4.1'), reason="Not implemented") |
| 173 |
@pytest.mark.skipif(get_default_db_lib() == "mdb", reason="Not supported over mdb") |
| 174 |
def test_db_home_dir_online_backup(topo): |
| 175 |
"""Test that if the dbhome directory is set causing an online backup to fail, |
| 176 |
the dblayer_backup function should go to error processing section. |
| 177 |
|
| 178 |
:id: cfc495d6-2a58-4e4e-aa40-39a15c71f973 |
| 179 |
:setup: Standalone Instance |
| 180 |
:steps: |
| 181 |
1. Change the dbhome to directory to eg-/tmp/test |
| 182 |
2. Perform an online back-up |
| 183 |
3. Check for the correct errors in the log |
| 184 |
:expectedresults: |
| 185 |
1. Success |
| 186 |
2. Failure |
| 187 |
3. Success |
| 188 |
""" |
| 189 |
inst = topo.standalone |
| 190 |
dn = BDB_LDBMConfig(inst).dn |
| 191 |
attr = 'nsslapd-db-home-directory' |
| 192 |
with tempfile.TemporaryDirectory() as dbhome_dir: |
| 193 |
with DseConfigContextManager(inst, dn, attr, dbhome_dir): |
| 194 |
backup_dir = str(dbhome_dir) |
| 195 |
inst.tasks.db2bak(backup_dir=backup_dir, args={TASK_WAIT: True}) |
| 196 |
assert inst.ds_error_log.match(f".*Failed renaming {backup_dir}.bak back to {backup_dir}") |
| 197 |
|
| 198 |
|
| 199 |
|
| 200 |
def test_replication(topo_m2): |
| 201 |
"""Test that if the dbhome directory is set causing an online backup to fail, |
| 202 |
the dblayer_backup function should go to error processing section. |
| 203 |
|
| 204 |
:id: 9c826d36-b17d-11ee-855f-482ae39447e5 |
| 205 |
:setup: Two suppliers |
| 206 |
:steps: |
| 207 |
1. Perform backup on S1 |
| 208 |
2. Perform changes on both suppliers |
| 209 |
3. Wait until replication is in sync |
| 210 |
4. Stop S1 |
| 211 |
5. Destroy S1 database |
| 212 |
6. Start S1 |
| 213 |
7. Restore S1 from backup |
| 214 |
8. Wait until replication is in sync |
| 215 |
:expectedresults: |
| 216 |
1. Success |
| 217 |
2. Success |
| 218 |
3. Success |
| 219 |
4. Success |
| 220 |
5. Success |
| 221 |
6. Success |
| 222 |
7. Success |
| 223 |
8. Success |
| 224 |
""" |
| 225 |
S1 = topo_m2.ms["supplier1"] |
| 226 |
S2 = topo_m2.ms["supplier2"] |
| 227 |
repl = ReplicationManager(DEFAULT_SUFFIX) |
| 228 |
|
| 229 |
with tempfile.TemporaryDirectory(dir=S1.ds_paths.backup_dir) as backup_dir: |
| 230 |
# Step 1: Perform backup on S1 |
| 231 |
# Use the offline method to have a cleanly stopped state in changelog. |
| 232 |
S1.stop() |
| 233 |
assert S1.db2bak(backup_dir) |
| 234 |
S1.start() |
| 235 |
|
| 236 |
# Step 2: Perform changes on both suppliers and wait for replication |
| 237 |
# Note: wait_for_replication perform changes |
| 238 |
repl.wait_for_replication(S1, S2) |
| 239 |
repl.wait_for_replication(S2, S1) |
| 240 |
# Step 4: Stop S1 |
| 241 |
S1.stop() |
| 242 |
# Step 5: Destroy S1 database |
| 243 |
if get_default_db_lib() == "mdb": |
| 244 |
os.remove(f'{S1.ds_paths.db_dir}/data.mdb') |
| 245 |
else: |
| 246 |
shutil.rmtree(f'{S1.ds_paths.db_dir}/userRoot') |
| 247 |
# Step 6: Start S1 |
| 248 |
S1.start() |
| 249 |
# Step 7: Restore S1 from backup |
| 250 |
rc = S1.tasks.bak2db(backup_dir=f'{backup_dir}', args={TASK_WAIT: True}) |
| 251 |
assert rc == 0 |
| 252 |
# Step 8: Wait until replication is in sync |
| 253 |
# Must replicate first from S2 to S1 to resync S1 |
| 254 |
repl.wait_for_replication(S2, S1) |
| 255 |
# To help to diagnose test failure, you may want to look first at: |
| 256 |
# grep -E 'Database RUV|replica_reload_ruv|task_restore_thread|_cl5ConstructRUVs' /var/log/dirsrv/slapd-supplier1/errors |
| 257 |
repl.wait_for_replication(S1, S2) |
| 258 |
|
| 259 |
|
| 260 |
def test_after_db_log_rotation(topo): |
| 261 |
"""Test that off line backup restore works as expected. |
| 262 |
|
| 263 |
:id: 8a091d92-a1cf-11ef-823a-482ae39447e5 |
| 264 |
:setup: One standalone instance |
| 265 |
:steps: |
| 266 |
1. Stop instance |
| 267 |
2. Perform off line backup on instance |
| 268 |
3. Start instance |
| 269 |
4. Perform modify operation until db log file rotates |
| 270 |
5. Stop instance |
| 271 |
6. Restore instance from backup |
| 272 |
7. Start instance |
| 273 |
:expectedresults: |
| 274 |
1. Success |
| 275 |
2. Success |
| 276 |
3. Success |
| 277 |
4. Success |
| 278 |
5. Success |
| 279 |
6. Success |
| 280 |
7. Success |
| 281 |
""" |
| 282 |
inst = topo.standalone |
| 283 |
with tempfile.TemporaryDirectory(dir=inst.ds_paths.backup_dir) as backup_dir: |
| 284 |
# repl.wait_for_replication perform some changes and wait until they get replicated |
| 285 |
inst.stop() |
| 286 |
assert inst.db2bak(backup_dir) |
| 287 |
inst.start() |
| 288 |
cmd = [ 'ldclt', '-h', 'localhost', '-b', DEFAULT_SUFFIX, |
| 289 |
'-p', str(inst.port), '-t', '60', '-N', '2', |
| 290 |
'-D', DN_DM, '-w', PASSWORD, '-f', "ou=People", |
| 291 |
'-e', 'attreplace=description:XXXXXX' ] |
| 292 |
log.info(f'Running {cmd}') |
| 293 |
# Perform modify operations until log file rolls |
| 294 |
result = subprocess.run(cmd, capture_output=True, text=True, check=True) |
| 295 |
log.info(f'STDOUT: {result.stdout}') |
| 296 |
log.info(f'STDERR: {result.stderr}') |
| 297 |
if get_default_db_lib() == 'bdb': |
| 298 |
while os.path.isfile(f'{inst.ds_paths.db_dir}/log.0000000001'): |
| 299 |
subprocess.run(cmd, capture_output=True, text=True, check=True) |
| 300 |
inst.stop() |
| 301 |
assert inst.bak2db(backup_dir) |
| 302 |
inst.start() |
| 303 |
|
| 304 |
|
| 305 |
def test_backup_task_after_failure(mytopo): |
| 306 |
"""Test that new backup task is successful after a failure. |
| 307 |
backend that is no longer present. |
| 308 |
|
| 309 |
:id: a6c24898-2cd9-11ef-8c09-482ae39447e5 |
| 310 |
:setup: Standalone Instance with multiple backends |
| 311 |
:steps: |
| 312 |
1. Cleanup |
| 313 |
2. Perform a back up |
| 314 |
3. Rename the backup directory while waiting for backup completion. |
| 315 |
4. Check that backup failed. |
| 316 |
5. Perform a back up |
| 317 |
6. Check that backup succeed. |
| 318 |
:expectedresults: |
| 319 |
1. Success |
| 320 |
2. Success |
| 321 |
3. Success |
| 322 |
4. Backup should fail |
| 323 |
5. Success |
| 324 |
6. Backup should succeed |
| 325 |
""" |
| 326 |
|
| 327 |
inst = mytopo.standalone |
| 328 |
tasks = inst.tasks |
| 329 |
archive_dir1 = f'{inst.ds_paths.backup_dir}/bak1' |
| 330 |
archive_dir1b = f'{inst.ds_paths.backup_dir}/bak1b' |
| 331 |
archive_dir2 = f'{inst.ds_paths.backup_dir}/bak2' |
| 332 |
|
| 333 |
# Sometime the backup complete too fast, so lets retry if first |
| 334 |
# backup is successful |
| 335 |
for retry in range(50): |
| 336 |
# Step 1. Perform cleanup |
| 337 |
for dir in glob.glob(f'{inst.ds_paths.backup_dir}/*'): |
| 338 |
shutil.rmtree(dir) |
| 339 |
# Step 2. Perform a backup |
| 340 |
tasks.db2bak(backup_dir=archive_dir1) |
| 341 |
# Step 3. Wait until task is completed, trying to rename backup directory |
| 342 |
done,exitCode,warningCode = (False, None, None) |
| 343 |
while not done: |
| 344 |
if os.path.isdir(archive_dir1): |
| 345 |
os.rename(archive_dir1, archive_dir1b) |
| 346 |
done,exitCode,warningCode = tasks.checkTask(tasks.entry) |
| 347 |
time.sleep(0.01) |
| 348 |
if exitCode != 0: |
| 349 |
break |
| 350 |
# Step 4. Check that backup failed. |
| 351 |
# If next assert fails too often, that means that the backup is too fast |
| 352 |
# A fix would would probably be to add more backends within mytopo |
| 353 |
assert exitCode != 0, "Backup did not fail as expected." |
| 354 |
# Step 5. Perform a seconf backup after backup failure |
| 355 |
exitCode = tasks.db2bak(backup_dir=archive_dir2, args={TASK_WAIT: True}) |
| 356 |
# Step 6. Check it is successful |
| 357 |
assert exitCode == 0, "Backup failed. Issue #6229 may not be fixed." |
| 358 |
|
| 359 |
|
| 360 |
def load_dse(inst): |
| 361 |
conts = nsContainers(inst, 'cn=config') |
| 362 |
while not event.is_set(): |
| 363 |
cont = conts.create(properties={'cn': 'test_online_backup_and_dse_write'}) |
| 364 |
cont.delete() |
| 365 |
|
| 366 |
|
| 367 |
def test_online_backup_and_dse_write(topo): |
| 368 |
"""Test online backup while attempting to add/delete entries in dse.ldif. |
| 369 |
|
| 370 |
:id: 4a1edd2c-be15-11ef-8bc8-482ae39447e5 |
| 371 |
:setup: One standalone instance |
| 372 |
:steps: |
| 373 |
1. Start a thread that loops adding then removing in the dse.ldif |
| 374 |
2. Perform 10 online backups |
| 375 |
3. Stop the thread |
| 376 |
:expectedresults: |
| 377 |
1. Success |
| 378 |
2. Success (or timeout if issue #6372 is not fixed.) |
| 379 |
3. Success |
| 380 |
""" |
| 381 |
inst = topo.standalone |
| 382 |
t = Thread(target=load_dse, args=[inst]) |
| 383 |
t.start() |
| 384 |
for x in range(10): |
| 385 |
with tempfile.TemporaryDirectory() as backup_dir: |
| 386 |
assert inst.tasks.db2bak(backup_dir=f'{backup_dir}', args={TASK_WAIT: True}) == 0 |
| 387 |
event.set() |
| 388 |
t.join() |
| 389 |
event.clear() |
| 390 |
|
| 391 |
|
| 392 |
if __name__ == '__main__': |
| 393 |
# Run isolated |
| 394 |
# -s for DEBUG mode |
| 395 |
CURRENT_FILE = os.path.realpath(__file__) |
| 396 |
pytest.main(["-s", CURRENT_FILE]) |